1 /*
2 * Copyright © 2012 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "blorp_nir_builder.h"
25 #include "compiler/nir/nir_format_convert.h"
26
27 #include "blorp_priv.h"
28 #include "dev/intel_debug.h"
29 #include "dev/intel_device_info.h"
30
31 #include "util/format_rgb9e5.h"
32 #include "util/u_math.h"
33
34 #define FILE_DEBUG_FLAG DEBUG_BLORP
35
36 static const bool split_blorp_blit_debug = false;
37
38 struct blorp_blit_vars {
39 /* Input values from blorp_wm_inputs */
40 nir_variable *v_bounds_rect;
41 nir_variable *v_rect_grid;
42 nir_variable *v_coord_transform;
43 nir_variable *v_src_z;
44 nir_variable *v_src_offset;
45 nir_variable *v_dst_offset;
46 nir_variable *v_src_inv_size;
47 };
48
49 static void
blorp_blit_vars_init(nir_builder * b,struct blorp_blit_vars * v,const struct blorp_blit_prog_key * key)50 blorp_blit_vars_init(nir_builder *b, struct blorp_blit_vars *v,
51 const struct blorp_blit_prog_key *key)
52 {
53 #define LOAD_INPUT(name, type)\
54 v->v_##name = BLORP_CREATE_NIR_INPUT(b->shader, name, type);
55
56 LOAD_INPUT(bounds_rect, glsl_vec4_type())
57 LOAD_INPUT(rect_grid, glsl_vec4_type())
58 LOAD_INPUT(coord_transform, glsl_vec4_type())
59 LOAD_INPUT(src_z, glsl_float_type())
60 LOAD_INPUT(src_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
61 LOAD_INPUT(dst_offset, glsl_vector_type(GLSL_TYPE_UINT, 2))
62 LOAD_INPUT(src_inv_size, glsl_vector_type(GLSL_TYPE_FLOAT, 2))
63
64 #undef LOAD_INPUT
65 }
66
67 static nir_def *
blorp_blit_get_frag_coords(nir_builder * b,const struct blorp_blit_prog_key * key,struct blorp_blit_vars * v)68 blorp_blit_get_frag_coords(nir_builder *b,
69 const struct blorp_blit_prog_key *key,
70 struct blorp_blit_vars *v)
71 {
72 nir_def *coord = nir_f2i32(b, nir_load_frag_coord(b));
73
74 /* Account for destination surface intratile offset
75 *
76 * Transformation parameters giving translation from destination to source
77 * coordinates don't take into account possible intra-tile destination
78 * offset. Therefore it has to be first subtracted from the incoming
79 * coordinates. Vertices are set up based on coordinates containing the
80 * intra-tile offset.
81 */
82 if (key->need_dst_offset)
83 coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));
84
85 if (key->persample_msaa_dispatch) {
86 b->shader->info.fs.uses_sample_shading = true;
87 return nir_vec3(b, nir_channel(b, coord, 0), nir_channel(b, coord, 1),
88 nir_load_sample_id(b));
89 } else {
90 return nir_trim_vector(b, coord, 2);
91 }
92 }
93
94 static nir_def *
blorp_blit_get_cs_dst_coords(nir_builder * b,const struct blorp_blit_prog_key * key,struct blorp_blit_vars * v)95 blorp_blit_get_cs_dst_coords(nir_builder *b,
96 const struct blorp_blit_prog_key *key,
97 struct blorp_blit_vars *v)
98 {
99 nir_def *coord = nir_load_global_invocation_id(b, 32);
100
101 /* Account for destination surface intratile offset
102 *
103 * Transformation parameters giving translation from destination to source
104 * coordinates don't take into account possible intra-tile destination
105 * offset. Therefore it has to be first subtracted from the incoming
106 * coordinates. Vertices are set up based on coordinates containing the
107 * intra-tile offset.
108 */
109 if (key->need_dst_offset)
110 coord = nir_isub(b, coord, nir_load_var(b, v->v_dst_offset));
111
112 assert(!key->persample_msaa_dispatch);
113 return nir_trim_vector(b, coord, 2);
114 }
115
116 /**
117 * Emit code to translate from destination (X, Y) coordinates to source (X, Y)
118 * coordinates.
119 */
120 static nir_def *
blorp_blit_apply_transform(nir_builder * b,nir_def * src_pos,struct blorp_blit_vars * v)121 blorp_blit_apply_transform(nir_builder *b, nir_def *src_pos,
122 struct blorp_blit_vars *v)
123 {
124 nir_def *coord_transform = nir_load_var(b, v->v_coord_transform);
125
126 nir_def *offset = nir_vec2(b, nir_channel(b, coord_transform, 1),
127 nir_channel(b, coord_transform, 3));
128 nir_def *mul = nir_vec2(b, nir_channel(b, coord_transform, 0),
129 nir_channel(b, coord_transform, 2));
130
131 return nir_fadd(b, nir_fmul(b, src_pos, mul), offset);
132 }
133
134 static nir_tex_instr *
blorp_create_nir_tex_instr(nir_builder * b,struct blorp_blit_vars * v,nir_texop op,nir_def * pos,unsigned num_srcs,nir_alu_type dst_type)135 blorp_create_nir_tex_instr(nir_builder *b, struct blorp_blit_vars *v,
136 nir_texop op, nir_def *pos, unsigned num_srcs,
137 nir_alu_type dst_type)
138 {
139 nir_tex_instr *tex = nir_tex_instr_create(b->shader, num_srcs);
140
141 tex->op = op;
142
143 tex->dest_type = dst_type | 32;
144 tex->is_array = false;
145 tex->is_shadow = false;
146
147 tex->texture_index = BLORP_TEXTURE_BT_INDEX;
148 tex->sampler_index = BLORP_SAMPLER_INDEX;
149
150 /* To properly handle 3-D and 2-D array textures, we pull the Z component
151 * from an input. TODO: This is a bit magic; we should probably make this
152 * more explicit in the future.
153 */
154 assert(pos->num_components >= 2);
155 if (op == nir_texop_txf || op == nir_texop_txf_ms ||
156 op == nir_texop_txf_ms_mcs_intel) {
157 pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
158 nir_f2i32(b, nir_load_var(b, v->v_src_z)));
159 } else {
160 pos = nir_vec3(b, nir_channel(b, pos, 0), nir_channel(b, pos, 1),
161 nir_load_var(b, v->v_src_z));
162 }
163
164 tex->src[0] = nir_tex_src_for_ssa(nir_tex_src_coord, pos);
165 tex->coord_components = 3;
166
167 nir_def_init(&tex->instr, &tex->def, 4, 32);
168
169 return tex;
170 }
171
172 static nir_def *
blorp_nir_tex(nir_builder * b,struct blorp_blit_vars * v,const struct blorp_blit_prog_key * key,nir_def * pos)173 blorp_nir_tex(nir_builder *b, struct blorp_blit_vars *v,
174 const struct blorp_blit_prog_key *key, nir_def *pos)
175 {
176 if (key->need_src_offset)
177 pos = nir_fadd(b, pos, nir_i2f32(b, nir_load_var(b, v->v_src_offset)));
178
179 /* If the sampler requires normalized coordinates, we need to compensate. */
180 if (key->src_coords_normalized)
181 pos = nir_fmul(b, pos, nir_load_var(b, v->v_src_inv_size));
182
183 nir_tex_instr *tex =
184 blorp_create_nir_tex_instr(b, v, nir_texop_txl, pos, 2,
185 key->texture_data_type);
186
187 assert(pos->num_components == 2);
188 tex->sampler_dim = GLSL_SAMPLER_DIM_2D;
189 tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(b, 0));
190
191 nir_builder_instr_insert(b, &tex->instr);
192
193 return &tex->def;
194 }
195
196 static nir_def *
blorp_nir_txf(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos,nir_alu_type dst_type)197 blorp_nir_txf(nir_builder *b, struct blorp_blit_vars *v,
198 nir_def *pos, nir_alu_type dst_type)
199 {
200 nir_tex_instr *tex =
201 blorp_create_nir_tex_instr(b, v, nir_texop_txf, pos, 2, dst_type);
202
203 tex->sampler_dim = GLSL_SAMPLER_DIM_3D;
204 tex->src[1] = nir_tex_src_for_ssa(nir_tex_src_lod, nir_imm_int(b, 0));
205
206 nir_builder_instr_insert(b, &tex->instr);
207
208 return &tex->def;
209 }
210
211 static nir_def *
blorp_nir_txf_ms(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos,nir_def * mcs,nir_alu_type dst_type)212 blorp_nir_txf_ms(nir_builder *b, struct blorp_blit_vars *v,
213 nir_def *pos, nir_def *mcs, nir_alu_type dst_type)
214 {
215 nir_tex_instr *tex =
216 blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms, pos, 3, dst_type);
217
218 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
219
220 tex->src[1].src_type = nir_tex_src_ms_index;
221 if (pos->num_components == 2) {
222 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0));
223 } else {
224 assert(pos->num_components == 3);
225 tex->src[1].src = nir_src_for_ssa(nir_channel(b, pos, 2));
226 }
227
228 if (!mcs)
229 mcs = nir_imm_zero(b, 4, 32);
230
231 tex->src[2] = nir_tex_src_for_ssa(nir_tex_src_ms_mcs_intel, mcs);
232
233 nir_builder_instr_insert(b, &tex->instr);
234
235 return &tex->def;
236 }
237
238 static nir_def *
blorp_blit_txf_ms_mcs(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos)239 blorp_blit_txf_ms_mcs(nir_builder *b, struct blorp_blit_vars *v,
240 nir_def *pos)
241 {
242 nir_tex_instr *tex =
243 blorp_create_nir_tex_instr(b, v, nir_texop_txf_ms_mcs_intel,
244 pos, 1, nir_type_int);
245
246 tex->sampler_dim = GLSL_SAMPLER_DIM_MS;
247
248 nir_builder_instr_insert(b, &tex->instr);
249
250 return &tex->def;
251 }
252
253 /**
254 * Emit code to compensate for the difference between Y and W tiling.
255 *
256 * This code modifies the X and Y coordinates according to the formula:
257 *
258 * (X', Y', S') = detile(W-MAJOR, tile(Y-MAJOR, X, Y, S))
259 *
260 * (See blorp_build_nir_shader).
261 */
262 static inline nir_def *
blorp_nir_retile_y_to_w(nir_builder * b,nir_def * pos)263 blorp_nir_retile_y_to_w(nir_builder *b, nir_def *pos)
264 {
265 assert(pos->num_components == 2);
266 nir_def *x_Y = nir_channel(b, pos, 0);
267 nir_def *y_Y = nir_channel(b, pos, 1);
268
269 /* Given X and Y coordinates that describe an address using Y tiling,
270 * translate to the X and Y coordinates that describe the same address
271 * using W tiling.
272 *
273 * If we break down the low order bits of X and Y, using a
274 * single letter to represent each low-order bit:
275 *
276 * X = A << 7 | 0bBCDEFGH
277 * Y = J << 5 | 0bKLMNP (1)
278 *
279 * Then we can apply the Y tiling formula to see the memory offset being
280 * addressed:
281 *
282 * offset = (J * tile_pitch + A) << 12 | 0bBCDKLMNPEFGH (2)
283 *
284 * If we apply the W detiling formula to this memory location, that the
285 * corresponding X' and Y' coordinates are:
286 *
287 * X' = A << 6 | 0bBCDPFH (3)
288 * Y' = J << 6 | 0bKLMNEG
289 *
290 * Combining (1) and (3), we see that to transform (X, Y) to (X', Y'),
291 * we need to make the following computation:
292 *
293 * X' = (X & ~0b1011) >> 1 | (Y & 0b1) << 2 | X & 0b1 (4)
294 * Y' = (Y & ~0b1) << 1 | (X & 0b1000) >> 2 | (X & 0b10) >> 1
295 */
296 nir_def *x_W = nir_imm_int(b, 0);
297 x_W = nir_mask_shift_or(b, x_W, x_Y, 0xfffffff4, -1);
298 x_W = nir_mask_shift_or(b, x_W, y_Y, 0x1, 2);
299 x_W = nir_mask_shift_or(b, x_W, x_Y, 0x1, 0);
300
301 nir_def *y_W = nir_imm_int(b, 0);
302 y_W = nir_mask_shift_or(b, y_W, y_Y, 0xfffffffe, 1);
303 y_W = nir_mask_shift_or(b, y_W, x_Y, 0x8, -2);
304 y_W = nir_mask_shift_or(b, y_W, x_Y, 0x2, -1);
305
306 return nir_vec2(b, x_W, y_W);
307 }
308
309 /**
310 * Emit code to compensate for the difference between Y and W tiling.
311 *
312 * This code modifies the X and Y coordinates according to the formula:
313 *
314 * (X', Y', S') = detile(Y-MAJOR, tile(W-MAJOR, X, Y, S))
315 *
316 * (See blorp_build_nir_shader).
317 */
318 static inline nir_def *
blorp_nir_retile_w_to_y(nir_builder * b,nir_def * pos)319 blorp_nir_retile_w_to_y(nir_builder *b, nir_def *pos)
320 {
321 assert(pos->num_components == 2);
322 nir_def *x_W = nir_channel(b, pos, 0);
323 nir_def *y_W = nir_channel(b, pos, 1);
324
325 /* Applying the same logic as above, but in reverse, we obtain the
326 * formulas:
327 *
328 * X' = (X & ~0b101) << 1 | (Y & 0b10) << 2 | (Y & 0b1) << 1 | X & 0b1
329 * Y' = (Y & ~0b11) >> 1 | (X & 0b100) >> 2
330 */
331 nir_def *x_Y = nir_imm_int(b, 0);
332 x_Y = nir_mask_shift_or(b, x_Y, x_W, 0xfffffffa, 1);
333 x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x2, 2);
334 x_Y = nir_mask_shift_or(b, x_Y, y_W, 0x1, 1);
335 x_Y = nir_mask_shift_or(b, x_Y, x_W, 0x1, 0);
336
337 nir_def *y_Y = nir_imm_int(b, 0);
338 y_Y = nir_mask_shift_or(b, y_Y, y_W, 0xfffffffc, -1);
339 y_Y = nir_mask_shift_or(b, y_Y, x_W, 0x4, -2);
340
341 return nir_vec2(b, x_Y, y_Y);
342 }
343
344 /**
345 * Emit code to compensate for the difference between MSAA and non-MSAA
346 * surfaces.
347 *
348 * This code modifies the X and Y coordinates according to the formula:
349 *
350 * (X', Y', S') = encode_msaa(num_samples, IMS, X, Y, S)
351 */
352 static inline nir_def *
blorp_nir_encode_msaa(nir_builder * b,nir_def * pos,unsigned num_samples,enum isl_msaa_layout layout)353 blorp_nir_encode_msaa(nir_builder *b, nir_def *pos,
354 unsigned num_samples, enum isl_msaa_layout layout)
355 {
356 assert(pos->num_components == 2 || pos->num_components == 3);
357
358 switch (layout) {
359 case ISL_MSAA_LAYOUT_NONE:
360 assert(pos->num_components == 2);
361 return pos;
362 case ISL_MSAA_LAYOUT_ARRAY:
363 /* No translation needed */
364 return pos;
365 case ISL_MSAA_LAYOUT_INTERLEAVED: {
366 nir_def *x_in = nir_channel(b, pos, 0);
367 nir_def *y_in = nir_channel(b, pos, 1);
368 nir_def *s_in = pos->num_components == 2 ? nir_imm_int(b, 0) :
369 nir_channel(b, pos, 2);
370
371 nir_def *x_out = nir_imm_int(b, 0);
372 nir_def *y_out = nir_imm_int(b, 0);
373 switch (num_samples) {
374 case 2:
375 case 4:
376 /* encode_msaa(2, IMS, X, Y, S) = (X', Y', 0)
377 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
378 * Y' = Y
379 *
380 * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
381 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
382 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
383 */
384 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 1);
385 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
386 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
387 if (num_samples == 2) {
388 y_out = y_in;
389 } else {
390 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
391 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
392 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
393 }
394 break;
395
396 case 8:
397 /* encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
398 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
399 * | (X & 0b1)
400 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
401 */
402 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
403 x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
404 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
405 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
406 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 1);
407 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
408 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
409 break;
410
411 case 16:
412 /* encode_msaa(16, IMS, X, Y, S) = (X', Y', 0)
413 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1
414 * | (X & 0b1)
415 * Y' = (Y & ~0b1) << 2 | (S & 0b1000) >> 1 (S & 0b10)
416 * | (Y & 0b1)
417 */
418 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffe, 2);
419 x_out = nir_mask_shift_or(b, x_out, s_in, 0x4, 0);
420 x_out = nir_mask_shift_or(b, x_out, s_in, 0x1, 1);
421 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
422 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffe, 2);
423 y_out = nir_mask_shift_or(b, y_out, s_in, 0x8, -1);
424 y_out = nir_mask_shift_or(b, y_out, s_in, 0x2, 0);
425 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
426 break;
427
428 default:
429 unreachable("Invalid number of samples for IMS layout");
430 }
431
432 return nir_vec2(b, x_out, y_out);
433 }
434
435 default:
436 unreachable("Invalid MSAA layout");
437 }
438 }
439
440 /**
441 * Emit code to compensate for the difference between MSAA and non-MSAA
442 * surfaces.
443 *
444 * This code modifies the X and Y coordinates according to the formula:
445 *
446 * (X', Y', S) = decode_msaa(num_samples, IMS, X, Y, S)
447 */
448 static inline nir_def *
blorp_nir_decode_msaa(nir_builder * b,nir_def * pos,unsigned num_samples,enum isl_msaa_layout layout)449 blorp_nir_decode_msaa(nir_builder *b, nir_def *pos,
450 unsigned num_samples, enum isl_msaa_layout layout)
451 {
452 assert(pos->num_components == 2 || pos->num_components == 3);
453
454 switch (layout) {
455 case ISL_MSAA_LAYOUT_NONE:
456 /* No translation necessary, and S should already be zero. */
457 assert(pos->num_components == 2);
458 return pos;
459 case ISL_MSAA_LAYOUT_ARRAY:
460 /* No translation necessary. */
461 return pos;
462 case ISL_MSAA_LAYOUT_INTERLEAVED: {
463 assert(pos->num_components == 2);
464
465 nir_def *x_in = nir_channel(b, pos, 0);
466 nir_def *y_in = nir_channel(b, pos, 1);
467
468 nir_def *x_out = nir_imm_int(b, 0);
469 nir_def *y_out = nir_imm_int(b, 0);
470 nir_def *s_out = nir_imm_int(b, 0);
471 switch (num_samples) {
472 case 2:
473 case 4:
474 /* decode_msaa(2, IMS, X, Y, 0) = (X', Y', S)
475 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
476 * S = (X & 0b10) >> 1
477 *
478 * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
479 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
480 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
481 * S = (Y & 0b10) | (X & 0b10) >> 1
482 */
483 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffffc, -1);
484 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
485 if (num_samples == 2) {
486 y_out = y_in;
487 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
488 } else {
489 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
490 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
491 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
492 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
493 }
494 break;
495
496 case 8:
497 /* decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
498 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
499 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
500 * S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
501 */
502 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
503 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
504 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffffc, -1);
505 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
506 s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
507 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
508 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
509 break;
510
511 case 16:
512 /* decode_msaa(16, IMS, X, Y, 0) = (X', Y', S)
513 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
514 * Y' = (Y & ~0b111) >> 2 | (Y & 0b1)
515 * S = (Y & 0b100) << 1 | (X & 0b100) |
516 * (Y & 0b10) | (X & 0b10) >> 1
517 */
518 x_out = nir_mask_shift_or(b, x_out, x_in, 0xfffffff8, -2);
519 x_out = nir_mask_shift_or(b, x_out, x_in, 0x1, 0);
520 y_out = nir_mask_shift_or(b, y_out, y_in, 0xfffffff8, -2);
521 y_out = nir_mask_shift_or(b, y_out, y_in, 0x1, 0);
522 s_out = nir_mask_shift_or(b, s_out, y_in, 0x4, 1);
523 s_out = nir_mask_shift_or(b, s_out, x_in, 0x4, 0);
524 s_out = nir_mask_shift_or(b, s_out, y_in, 0x2, 0);
525 s_out = nir_mask_shift_or(b, s_out, x_in, 0x2, -1);
526 break;
527
528 default:
529 unreachable("Invalid number of samples for IMS layout");
530 }
531
532 return nir_vec3(b, x_out, y_out, s_out);
533 }
534
535 default:
536 unreachable("Invalid MSAA layout");
537 }
538 }
539
540 /**
541 * Count the number of trailing 1 bits in the given value. For example:
542 *
543 * count_trailing_one_bits(0) == 0
544 * count_trailing_one_bits(7) == 3
545 * count_trailing_one_bits(11) == 2
546 */
count_trailing_one_bits(unsigned value)547 static inline int count_trailing_one_bits(unsigned value)
548 {
549 #ifdef HAVE___BUILTIN_CTZ
550 return __builtin_ctz(~value);
551 #else
552 return util_bitcount(value & ~(value + 1));
553 #endif
554 }
555
556 static nir_def *
blorp_nir_combine_samples(nir_builder * b,struct blorp_blit_vars * v,nir_def * pos,unsigned tex_samples,enum isl_aux_usage tex_aux_usage,nir_alu_type dst_type,enum blorp_filter filter)557 blorp_nir_combine_samples(nir_builder *b, struct blorp_blit_vars *v,
558 nir_def *pos, unsigned tex_samples,
559 enum isl_aux_usage tex_aux_usage,
560 nir_alu_type dst_type,
561 enum blorp_filter filter)
562 {
563 nir_variable *color =
564 nir_local_variable_create(b->impl, glsl_vec4_type(), "color");
565
566 nir_def *mcs = NULL;
567 if (isl_aux_usage_has_mcs(tex_aux_usage))
568 mcs = blorp_blit_txf_ms_mcs(b, v, pos);
569
570 nir_op combine_op;
571 switch (filter) {
572 case BLORP_FILTER_AVERAGE:
573 assert(dst_type == nir_type_float);
574 combine_op = nir_op_fadd;
575 break;
576
577 case BLORP_FILTER_MIN_SAMPLE:
578 switch (dst_type) {
579 case nir_type_int: combine_op = nir_op_imin; break;
580 case nir_type_uint: combine_op = nir_op_umin; break;
581 case nir_type_float: combine_op = nir_op_fmin; break;
582 default: unreachable("Invalid dst_type");
583 }
584 break;
585
586 case BLORP_FILTER_MAX_SAMPLE:
587 switch (dst_type) {
588 case nir_type_int: combine_op = nir_op_imax; break;
589 case nir_type_uint: combine_op = nir_op_umax; break;
590 case nir_type_float: combine_op = nir_op_fmax; break;
591 default: unreachable("Invalid dst_type");
592 }
593 break;
594
595 default:
596 unreachable("Invalid filter");
597 }
598
599 /* If true, we inserted an if statement that we need to pop at at the end.
600 */
601 bool inserted_if = false;
602
603 /* We add together samples using a binary tree structure, e.g. for 4x MSAA:
604 *
605 * result = ((sample[0] + sample[1]) + (sample[2] + sample[3])) / 4
606 *
607 * This ensures that when all samples have the same value, no numerical
608 * precision is lost, since each addition operation always adds two equal
609 * values, and summing two equal floating point values does not lose
610 * precision.
611 *
612 * We perform this computation by treating the texture_data array as a
613 * stack and performing the following operations:
614 *
615 * - push sample 0 onto stack
616 * - push sample 1 onto stack
617 * - add top two stack entries
618 * - push sample 2 onto stack
619 * - push sample 3 onto stack
620 * - add top two stack entries
621 * - add top two stack entries
622 * - divide top stack entry by 4
623 *
624 * Note that after pushing sample i onto the stack, the number of add
625 * operations we do is equal to the number of trailing 1 bits in i. This
626 * works provided the total number of samples is a power of two, which it
627 * always is for i965.
628 *
629 * For integer formats, we replace the add operations with average
630 * operations and skip the final division.
631 */
632 nir_def *texture_data[5];
633 texture_data[0] = NULL; /* Avoid maybe-uninitialized warning with GCC 10 */
634 unsigned stack_depth = 0;
635 for (unsigned i = 0; i < tex_samples; ++i) {
636 assert(stack_depth == util_bitcount(i)); /* Loop invariant */
637
638 /* Push sample i onto the stack */
639 assert(stack_depth < ARRAY_SIZE(texture_data));
640
641 nir_def *ms_pos = nir_vec3(b, nir_channel(b, pos, 0),
642 nir_channel(b, pos, 1),
643 nir_imm_int(b, i));
644 texture_data[stack_depth++] = blorp_nir_txf_ms(b, v, ms_pos, mcs, dst_type);
645
646 if (i == 0 && isl_aux_usage_has_mcs(tex_aux_usage)) {
647 /* The Ivy Bridge PRM, Vol4 Part1 p27 (Multisample Control Surface)
648 * suggests an optimization:
649 *
650 * "A simple optimization with probable large return in
651 * performance is to compare the MCS value to zero (indicating
652 * all samples are on sample slice 0), and sample only from
653 * sample slice 0 using ld2dss if MCS is zero."
654 *
655 * Note that in the case where the MCS value is zero, sampling from
656 * sample slice 0 using ld2dss and sampling from sample 0 using
657 * ld2dms are equivalent (since all samples are on sample slice 0).
658 * Since we have already sampled from sample 0, all we need to do is
659 * skip the remaining fetches and averaging if MCS is zero.
660 *
661 * It's also trivial to detect when the MCS has the magic clear color
662 * value. In this case, the txf we did on sample 0 will return the
663 * clear color and we can skip the remaining fetches just like we do
664 * when MCS == 0.
665 */
666 nir_def *mcs_zero = nir_ieq_imm(b, nir_channel(b, mcs, 0), 0);
667 if (tex_samples == 16) {
668 mcs_zero = nir_iand(b, mcs_zero,
669 nir_ieq_imm(b, nir_channel(b, mcs, 1), 0));
670 }
671 nir_def *mcs_clear =
672 blorp_nir_mcs_is_clear_color(b, mcs, tex_samples);
673
674 nir_push_if(b, nir_ior(b, mcs_zero, mcs_clear));
675 nir_store_var(b, color, texture_data[0], 0xf);
676
677 nir_push_else(b, NULL);
678 inserted_if = true;
679 }
680
681 for (int j = 0; j < count_trailing_one_bits(i); j++) {
682 assert(stack_depth >= 2);
683 --stack_depth;
684
685 texture_data[stack_depth - 1] =
686 nir_build_alu(b, combine_op,
687 texture_data[stack_depth - 1],
688 texture_data[stack_depth],
689 NULL, NULL);
690 }
691 }
692
693 /* We should have just 1 sample on the stack now. */
694 assert(stack_depth == 1);
695
696 if (filter == BLORP_FILTER_AVERAGE) {
697 assert(dst_type == nir_type_float);
698 texture_data[0] = nir_fmul_imm(b, texture_data[0],
699 1.0 / tex_samples);
700 }
701
702 nir_store_var(b, color, texture_data[0], 0xf);
703
704 if (inserted_if)
705 nir_pop_if(b, NULL);
706
707 return nir_load_var(b, color);
708 }
709
710 static nir_def *
blorp_nir_manual_blend_bilinear(nir_builder * b,nir_def * pos,unsigned tex_samples,const struct blorp_blit_prog_key * key,struct blorp_blit_vars * v)711 blorp_nir_manual_blend_bilinear(nir_builder *b, nir_def *pos,
712 unsigned tex_samples,
713 const struct blorp_blit_prog_key *key,
714 struct blorp_blit_vars *v)
715 {
716 nir_def *pos_xy = nir_trim_vector(b, pos, 2);
717 nir_def *rect_grid = nir_load_var(b, v->v_rect_grid);
718 nir_def *scale = nir_imm_vec2(b, key->x_scale, key->y_scale);
719
720 /* Translate coordinates to lay out the samples in a rectangular grid
721 * roughly corresponding to sample locations.
722 */
723 pos_xy = nir_fmul(b, pos_xy, scale);
724 /* Adjust coordinates so that integers represent pixel centers rather
725 * than pixel edges.
726 */
727 pos_xy = nir_fadd_imm(b, pos_xy, -0.5);
728 /* Clamp the X, Y texture coordinates to properly handle the sampling of
729 * texels on texture edges.
730 */
731 pos_xy = nir_fmin(b, nir_fmax(b, pos_xy, nir_imm_float(b, 0.0)),
732 nir_trim_vector(b, rect_grid, 2));
733
734 /* Store the fractional parts to be used as bilinear interpolation
735 * coefficients.
736 */
737 nir_def *frac_xy = nir_ffract(b, pos_xy);
738 /* Round the float coordinates down to nearest integer */
739 pos_xy = nir_fdiv(b, nir_ftrunc(b, pos_xy), scale);
740
741 nir_def *tex_data[4];
742 for (unsigned i = 0; i < 4; ++i) {
743 float sample_off_x = (float)(i & 0x1) / key->x_scale;
744 float sample_off_y = (float)((i >> 1) & 0x1) / key->y_scale;
745 nir_def *sample_off = nir_imm_vec2(b, sample_off_x, sample_off_y);
746
747 nir_def *sample_coords = nir_fadd(b, pos_xy, sample_off);
748 nir_def *sample_coords_int = nir_f2i32(b, sample_coords);
749
750 /* The MCS value we fetch has to match up with the pixel that we're
751 * sampling from. Since we sample from different pixels in each
752 * iteration of this "for" loop, the call to mcs_fetch() should be
753 * here inside the loop after computing the pixel coordinates.
754 */
755 nir_def *mcs = NULL;
756 if (isl_aux_usage_has_mcs(key->tex_aux_usage))
757 mcs = blorp_blit_txf_ms_mcs(b, v, sample_coords_int);
758
759 /* Compute sample index and map the sample index to a sample number.
760 * Sample index layout shows the numbering of slots in a rectangular
761 * grid of samples with in a pixel. Sample number layout shows the
762 * rectangular grid of samples roughly corresponding to the real sample
763 * locations with in a pixel.
764 *
765 * In the case of 2x MSAA, the layout of sample indices is reversed from
766 * the layout of sample numbers:
767 *
768 * sample index layout : --------- sample number layout : ---------
769 * | 0 | 1 | | 1 | 0 |
770 * --------- ---------
771 *
772 * In case of 4x MSAA, layout of sample indices matches the layout of
773 * sample numbers:
774 * ---------
775 * | 0 | 1 |
776 * ---------
777 * | 2 | 3 |
778 * ---------
779 *
780 * In case of 8x MSAA the two layouts don't match.
781 * sample index layout : --------- sample number layout : ---------
782 * | 0 | 1 | | 3 | 7 |
783 * --------- ---------
784 * | 2 | 3 | | 5 | 0 |
785 * --------- ---------
786 * | 4 | 5 | | 1 | 2 |
787 * --------- ---------
788 * | 6 | 7 | | 4 | 6 |
789 * --------- ---------
790 *
791 * Fortunately, this can be done fairly easily as:
792 * S' = (0x17306425 >> (S * 4)) & 0xf
793 *
794 * In the case of 16x MSAA the two layouts don't match.
795 * Sample index layout: Sample number layout:
796 * --------------------- ---------------------
797 * | 0 | 1 | 2 | 3 | | 15 | 10 | 9 | 7 |
798 * --------------------- ---------------------
799 * | 4 | 5 | 6 | 7 | | 4 | 1 | 3 | 13 |
800 * --------------------- ---------------------
801 * | 8 | 9 | 10 | 11 | | 12 | 2 | 0 | 6 |
802 * --------------------- ---------------------
803 * | 12 | 13 | 14 | 15 | | 11 | 8 | 5 | 14 |
804 * --------------------- ---------------------
805 *
806 * This is equivalent to
807 * S' = (0xe58b602cd31479af >> (S * 4)) & 0xf
808 */
809 nir_def *frac = nir_ffract(b, sample_coords);
810 nir_def *sample =
811 nir_fdot2(b, frac, nir_imm_vec2(b, key->x_scale,
812 key->x_scale * key->y_scale));
813 sample = nir_f2i32(b, sample);
814
815 if (tex_samples == 2) {
816 sample = nir_isub_imm(b, 1, sample);
817 } else if (tex_samples == 8) {
818 sample = nir_iand_imm(b, nir_ishr(b, nir_imm_int(b, 0x64210573),
819 nir_ishl_imm(b, sample, 2)),
820 0xf);
821 } else if (tex_samples == 16) {
822 nir_def *sample_low =
823 nir_iand_imm(b, nir_ishr(b, nir_imm_int(b, 0xd31479af),
824 nir_ishl_imm(b, sample, 2)),
825 0xf);
826 nir_def *sample_high =
827 nir_iand_imm(b, nir_ishr(b, nir_imm_int(b, 0xe58b602c),
828 nir_ishl_imm(b, nir_iadd_imm(b, sample, -8),
829 2)),
830 0xf);
831
832 sample = nir_bcsel(b, nir_ilt_imm(b, sample, 8),
833 sample_low, sample_high);
834 }
835 nir_def *pos_ms = nir_vec3(b, nir_channel(b, sample_coords_int, 0),
836 nir_channel(b, sample_coords_int, 1),
837 sample);
838 tex_data[i] = blorp_nir_txf_ms(b, v, pos_ms, mcs, key->texture_data_type);
839 }
840
841 nir_def *frac_x = nir_channel(b, frac_xy, 0);
842 nir_def *frac_y = nir_channel(b, frac_xy, 1);
843 return nir_flrp(b, nir_flrp(b, tex_data[0], tex_data[1], frac_x),
844 nir_flrp(b, tex_data[2], tex_data[3], frac_x),
845 frac_y);
846 }
847
848 /** Perform a color bit-cast operation
849 *
850 * For copy operations involving CCS, we may need to use different formats for
851 * the source and destination surfaces. The two formats must both be UINT
852 * formats and must have the same size but may have different bit layouts.
853 * For instance, we may be copying from R8G8B8A8_UINT to R32_UINT or R32_UINT
854 * to R16G16_UINT. This function generates code to shuffle bits around to get
855 * us from one to the other.
856 */
857 static nir_def *
bit_cast_color(struct nir_builder * b,nir_def * color,const struct blorp_blit_prog_key * key)858 bit_cast_color(struct nir_builder *b, nir_def *color,
859 const struct blorp_blit_prog_key *key)
860 {
861 if (key->src_format == key->dst_format)
862 return color;
863
864 const struct isl_format_layout *src_fmtl =
865 isl_format_get_layout(key->src_format);
866 const struct isl_format_layout *dst_fmtl =
867 isl_format_get_layout(key->dst_format);
868
869 /* They must be formats with the same bit size */
870 assert(src_fmtl->bpb == dst_fmtl->bpb);
871
872 if (src_fmtl->bpb <= 32) {
873 assert(src_fmtl->channels.r.type == ISL_UINT ||
874 src_fmtl->channels.r.type == ISL_UNORM);
875 assert(dst_fmtl->channels.r.type == ISL_UINT ||
876 dst_fmtl->channels.r.type == ISL_UNORM);
877
878 nir_def *packed = nir_imm_int(b, 0);
879 for (unsigned c = 0; c < 4; c++) {
880 if (src_fmtl->channels_array[c].bits == 0)
881 continue;
882
883 const unsigned chan_start_bit = src_fmtl->channels_array[c].start_bit;
884 const unsigned chan_bits = src_fmtl->channels_array[c].bits;
885
886 nir_def *chan = nir_channel(b, color, c);
887 if (src_fmtl->channels_array[c].type == ISL_UNORM)
888 chan = nir_format_float_to_unorm(b, chan, &chan_bits);
889
890 packed = nir_ior(b, packed, nir_shift_imm(b, chan, chan_start_bit));
891 }
892
893 nir_def *chans[4] = { };
894 for (unsigned c = 0; c < 4; c++) {
895 if (dst_fmtl->channels_array[c].bits == 0) {
896 chans[c] = nir_imm_int(b, 0);
897 continue;
898 }
899
900 const unsigned chan_start_bit = dst_fmtl->channels_array[c].start_bit;
901 const unsigned chan_bits = dst_fmtl->channels_array[c].bits;
902 chans[c] = nir_iand_imm(b, nir_shift_imm(b, packed, -(int)chan_start_bit),
903 BITFIELD_MASK(chan_bits));
904
905 if (dst_fmtl->channels_array[c].type == ISL_UNORM)
906 chans[c] = nir_format_unorm_to_float(b, chans[c], &chan_bits);
907 }
908 color = nir_vec(b, chans, 4);
909 } else {
910 /* This path only supports UINT formats */
911 assert(src_fmtl->channels.r.type == ISL_UINT);
912 assert(dst_fmtl->channels.r.type == ISL_UINT);
913
914 const unsigned src_bpc = src_fmtl->channels.r.bits;
915 const unsigned dst_bpc = dst_fmtl->channels.r.bits;
916
917 assert(src_fmtl->channels.g.bits == 0 ||
918 src_fmtl->channels.g.bits == src_fmtl->channels.r.bits);
919 assert(src_fmtl->channels.b.bits == 0 ||
920 src_fmtl->channels.b.bits == src_fmtl->channels.r.bits);
921 assert(src_fmtl->channels.a.bits == 0 ||
922 src_fmtl->channels.a.bits == src_fmtl->channels.r.bits);
923 assert(dst_fmtl->channels.g.bits == 0 ||
924 dst_fmtl->channels.g.bits == dst_fmtl->channels.r.bits);
925 assert(dst_fmtl->channels.b.bits == 0 ||
926 dst_fmtl->channels.b.bits == dst_fmtl->channels.r.bits);
927 assert(dst_fmtl->channels.a.bits == 0 ||
928 dst_fmtl->channels.a.bits == dst_fmtl->channels.r.bits);
929
930 /* Restrict to only the channels we actually have */
931 const unsigned src_channels =
932 isl_format_get_num_channels(key->src_format);
933 color = nir_trim_vector(b, color, src_channels);
934
935 color = nir_format_bitcast_uvec_unmasked(b, color, src_bpc, dst_bpc);
936 }
937
938 /* Blorp likes to assume that colors are vec4s */
939 nir_def *u = nir_undef(b, 1, 32);
940 nir_def *chans[4] = { u, u, u, u };
941 for (unsigned i = 0; i < color->num_components; i++)
942 chans[i] = nir_channel(b, color, i);
943 return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
944 }
945
946 static nir_def *
select_color_channel(struct nir_builder * b,nir_def * color,nir_alu_type data_type,enum isl_channel_select chan)947 select_color_channel(struct nir_builder *b, nir_def *color,
948 nir_alu_type data_type,
949 enum isl_channel_select chan)
950 {
951 if (chan == ISL_CHANNEL_SELECT_ZERO) {
952 return nir_imm_int(b, 0);
953 } else if (chan == ISL_CHANNEL_SELECT_ONE) {
954 switch (data_type) {
955 case nir_type_int:
956 case nir_type_uint:
957 return nir_imm_int(b, 1);
958 case nir_type_float:
959 return nir_imm_float(b, 1);
960 default:
961 unreachable("Invalid data type");
962 }
963 } else {
964 assert((unsigned)(chan - ISL_CHANNEL_SELECT_RED) < 4);
965 return nir_channel(b, color, chan - ISL_CHANNEL_SELECT_RED);
966 }
967 }
968
969 static nir_def *
swizzle_color(struct nir_builder * b,nir_def * color,struct isl_swizzle swizzle,nir_alu_type data_type)970 swizzle_color(struct nir_builder *b, nir_def *color,
971 struct isl_swizzle swizzle, nir_alu_type data_type)
972 {
973 return nir_vec4(b,
974 select_color_channel(b, color, data_type, swizzle.r),
975 select_color_channel(b, color, data_type, swizzle.g),
976 select_color_channel(b, color, data_type, swizzle.b),
977 select_color_channel(b, color, data_type, swizzle.a));
978 }
979
980 static nir_def *
convert_color(struct nir_builder * b,nir_def * color,const struct blorp_blit_prog_key * key)981 convert_color(struct nir_builder *b, nir_def *color,
982 const struct blorp_blit_prog_key *key)
983 {
984 /* All of our color conversions end up generating a single-channel color
985 * value that we need to write out.
986 */
987 nir_def *value;
988
989 if (key->dst_format == ISL_FORMAT_R24_UNORM_X8_TYPELESS) {
990 /* The destination image is bound as R32_UINT but the data needs to be
991 * in R24_UNORM_X8_TYPELESS. The bottom 24 are the actual data and the
992 * top 8 need to be zero. We can accomplish this by simply multiplying
993 * by a factor to scale things down.
994 */
995 unsigned factor = (1 << 24) - 1;
996 value = nir_fsat(b, nir_channel(b, color, 0));
997 value = nir_f2i32(b, nir_fmul_imm(b, value, factor));
998 } else if (key->dst_format == ISL_FORMAT_L8_UNORM_SRGB) {
999 value = nir_format_linear_to_srgb(b, nir_channel(b, color, 0));
1000 } else if (key->dst_format == ISL_FORMAT_R8G8B8_UNORM_SRGB) {
1001 value = nir_format_linear_to_srgb(b, color);
1002 } else if (key->dst_format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
1003 value = nir_format_pack_r9g9b9e5(b, color);
1004 } else {
1005 unreachable("Unsupported format conversion");
1006 }
1007
1008 nir_def *out_comps[4];
1009 for (unsigned i = 0; i < 4; i++) {
1010 if (i < value->num_components)
1011 out_comps[i] = nir_channel(b, value, i);
1012 else
1013 out_comps[i] = nir_undef(b, 1, 32);
1014 }
1015 return nir_vec(b, out_comps, 4);
1016 }
1017
1018 /**
1019 * Generator for WM programs used in BLORP blits.
1020 *
1021 * The bulk of the work done by the WM program is to wrap and unwrap the
1022 * coordinate transformations used by the hardware to store surfaces in
1023 * memory. The hardware transforms a pixel location (X, Y, S) (where S is the
1024 * sample index for a multisampled surface) to a memory offset by the
1025 * following formulas:
1026 *
1027 * offset = tile(tiling_format, encode_msaa(num_samples, layout, X, Y, S))
1028 * (X, Y, S) = decode_msaa(num_samples, layout, detile(tiling_format, offset))
1029 *
1030 * For a single-sampled surface, or for a multisampled surface using
1031 * INTEL_MSAA_LAYOUT_UMS, encode_msaa() and decode_msaa are the identity
1032 * function:
1033 *
1034 * encode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1035 * decode_msaa(1, NONE, X, Y, 0) = (X, Y, 0)
1036 * encode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1037 * decode_msaa(n, UMS, X, Y, S) = (X, Y, S)
1038 *
1039 * For a 4x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1040 * embeds the sample number into bit 1 of the X and Y coordinates:
1041 *
1042 * encode_msaa(4, IMS, X, Y, S) = (X', Y', 0)
1043 * where X' = (X & ~0b1) << 1 | (S & 0b1) << 1 | (X & 0b1)
1044 * Y' = (Y & ~0b1 ) << 1 | (S & 0b10) | (Y & 0b1)
1045 * decode_msaa(4, IMS, X, Y, 0) = (X', Y', S)
1046 * where X' = (X & ~0b11) >> 1 | (X & 0b1)
1047 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1048 * S = (Y & 0b10) | (X & 0b10) >> 1
1049 *
1050 * For an 8x multisampled surface using INTEL_MSAA_LAYOUT_IMS, encode_msaa()
1051 * embeds the sample number into bits 1 and 2 of the X coordinate and bit 1 of
1052 * the Y coordinate:
1053 *
1054 * encode_msaa(8, IMS, X, Y, S) = (X', Y', 0)
1055 * where X' = (X & ~0b1) << 2 | (S & 0b100) | (S & 0b1) << 1 | (X & 0b1)
1056 * Y' = (Y & ~0b1) << 1 | (S & 0b10) | (Y & 0b1)
1057 * decode_msaa(8, IMS, X, Y, 0) = (X', Y', S)
1058 * where X' = (X & ~0b111) >> 2 | (X & 0b1)
1059 * Y' = (Y & ~0b11) >> 1 | (Y & 0b1)
1060 * S = (X & 0b100) | (Y & 0b10) | (X & 0b10) >> 1
1061 *
1062 * For X tiling, tile() combines together the low-order bits of the X and Y
1063 * coordinates in the pattern 0byyyxxxxxxxxx, creating 4k tiles that are 512
1064 * bytes wide and 8 rows high:
1065 *
1066 * tile(x_tiled, X, Y, S) = A
1067 * where A = tile_num << 12 | offset
1068 * tile_num = (Y' >> 3) * tile_pitch + (X' >> 9)
1069 * offset = (Y' & 0b111) << 9
1070 * | (X & 0b111111111)
1071 * X' = X * cpp
1072 * Y' = Y + S * qpitch
1073 * detile(x_tiled, A) = (X, Y, S)
1074 * where X = X' / cpp
1075 * Y = Y' % qpitch
1076 * S = Y' / qpitch
1077 * Y' = (tile_num / tile_pitch) << 3
1078 * | (A & 0b111000000000) >> 9
1079 * X' = (tile_num % tile_pitch) << 9
1080 * | (A & 0b111111111)
1081 *
1082 * (In all tiling formulas, cpp is the number of bytes occupied by a single
1083 * sample ("chars per pixel"), tile_pitch is the number of 4k tiles required
1084 * to fill the width of the surface, and qpitch is the spacing (in rows)
1085 * between array slices).
1086 *
1087 * For Y tiling, tile() combines together the low-order bits of the X and Y
1088 * coordinates in the pattern 0bxxxyyyyyxxxx, creating 4k tiles that are 128
1089 * bytes wide and 32 rows high:
1090 *
1091 * tile(y_tiled, X, Y, S) = A
1092 * where A = tile_num << 12 | offset
1093 * tile_num = (Y' >> 5) * tile_pitch + (X' >> 7)
1094 * offset = (X' & 0b1110000) << 5
1095 * | (Y' & 0b11111) << 4
1096 * | (X' & 0b1111)
1097 * X' = X * cpp
1098 * Y' = Y + S * qpitch
1099 * detile(y_tiled, A) = (X, Y, S)
1100 * where X = X' / cpp
1101 * Y = Y' % qpitch
1102 * S = Y' / qpitch
1103 * Y' = (tile_num / tile_pitch) << 5
1104 * | (A & 0b111110000) >> 4
1105 * X' = (tile_num % tile_pitch) << 7
1106 * | (A & 0b111000000000) >> 5
1107 * | (A & 0b1111)
1108 *
1109 * For W tiling, tile() combines together the low-order bits of the X and Y
1110 * coordinates in the pattern 0bxxxyyyyxyxyx, creating 4k tiles that are 64
1111 * bytes wide and 64 rows high (note that W tiling is only used for stencil
1112 * buffers, which always have cpp = 1 and S=0):
1113 *
1114 * tile(w_tiled, X, Y, S) = A
1115 * where A = tile_num << 12 | offset
1116 * tile_num = (Y' >> 6) * tile_pitch + (X' >> 6)
1117 * offset = (X' & 0b111000) << 6
1118 * | (Y' & 0b111100) << 3
1119 * | (X' & 0b100) << 2
1120 * | (Y' & 0b10) << 2
1121 * | (X' & 0b10) << 1
1122 * | (Y' & 0b1) << 1
1123 * | (X' & 0b1)
1124 * X' = X * cpp = X
1125 * Y' = Y + S * qpitch
1126 * detile(w_tiled, A) = (X, Y, S)
1127 * where X = X' / cpp = X'
1128 * Y = Y' % qpitch = Y'
1129 * S = Y / qpitch = 0
1130 * Y' = (tile_num / tile_pitch) << 6
1131 * | (A & 0b111100000) >> 3
1132 * | (A & 0b1000) >> 2
1133 * | (A & 0b10) >> 1
1134 * X' = (tile_num % tile_pitch) << 6
1135 * | (A & 0b111000000000) >> 6
1136 * | (A & 0b10000) >> 2
1137 * | (A & 0b100) >> 1
1138 * | (A & 0b1)
1139 *
1140 * Finally, for a non-tiled surface, tile() simply combines together the X and
1141 * Y coordinates in the natural way:
1142 *
1143 * tile(untiled, X, Y, S) = A
1144 * where A = Y * pitch + X'
1145 * X' = X * cpp
1146 * Y' = Y + S * qpitch
1147 * detile(untiled, A) = (X, Y, S)
1148 * where X = X' / cpp
1149 * Y = Y' % qpitch
1150 * S = Y' / qpitch
1151 * X' = A % pitch
1152 * Y' = A / pitch
1153 *
1154 * (In these formulas, pitch is the number of bytes occupied by a single row
1155 * of samples).
1156 */
1157 static nir_shader *
blorp_build_nir_shader(struct blorp_context * blorp,struct blorp_batch * batch,void * mem_ctx,const struct blorp_blit_prog_key * key)1158 blorp_build_nir_shader(struct blorp_context *blorp,
1159 struct blorp_batch *batch, void *mem_ctx,
1160 const struct blorp_blit_prog_key *key)
1161 {
1162 const struct intel_device_info *devinfo = blorp->isl_dev->info;
1163 nir_def *src_pos, *dst_pos, *color;
1164
1165 /* Sanity checks */
1166 if (key->dst_tiled_w && key->rt_samples > 1) {
1167 /* If the destination image is W tiled and multisampled, then the thread
1168 * must be dispatched once per sample, not once per pixel. This is
1169 * necessary because after conversion between W and Y tiling, there's no
1170 * guarantee that all samples corresponding to a single pixel will still
1171 * be together.
1172 */
1173 assert(key->persample_msaa_dispatch);
1174 }
1175
1176 if (key->persample_msaa_dispatch) {
1177 /* It only makes sense to do persample dispatch if the render target is
1178 * configured as multisampled.
1179 */
1180 assert(key->rt_samples > 0);
1181 }
1182
1183 /* Make sure layout is consistent with sample count */
1184 assert((key->tex_layout == ISL_MSAA_LAYOUT_NONE) ==
1185 (key->tex_samples <= 1));
1186 assert((key->rt_layout == ISL_MSAA_LAYOUT_NONE) ==
1187 (key->rt_samples <= 1));
1188 assert((key->src_layout == ISL_MSAA_LAYOUT_NONE) ==
1189 (key->src_samples <= 1));
1190 assert((key->dst_layout == ISL_MSAA_LAYOUT_NONE) ==
1191 (key->dst_samples <= 1));
1192
1193 nir_builder b;
1194 const bool compute =
1195 key->base.shader_pipeline == BLORP_SHADER_PIPELINE_COMPUTE;
1196 gl_shader_stage stage =
1197 compute ? MESA_SHADER_COMPUTE : MESA_SHADER_FRAGMENT;
1198 blorp_nir_init_shader(&b, blorp, mem_ctx, stage, NULL);
1199
1200 struct blorp_blit_vars v;
1201 blorp_blit_vars_init(&b, &v, key);
1202
1203 dst_pos = compute ?
1204 blorp_blit_get_cs_dst_coords(&b, key, &v) :
1205 blorp_blit_get_frag_coords(&b, key, &v);
1206
1207 /* Render target and texture hardware don't support W tiling until Gfx8. */
1208 const bool rt_tiled_w = false;
1209 const bool tex_tiled_w = devinfo->ver >= 8 && key->src_tiled_w;
1210
1211 /* The address that data will be written to is determined by the
1212 * coordinates supplied to the WM thread and the tiling and sample count of
1213 * the render target, according to the formula:
1214 *
1215 * (X, Y, S) = decode_msaa(rt_samples, detile(rt_tiling, offset))
1216 *
1217 * If the actual tiling and sample count of the destination surface are not
1218 * the same as the configuration of the render target, then these
1219 * coordinates are wrong and we have to adjust them to compensate for the
1220 * difference.
1221 */
1222 if (rt_tiled_w != key->dst_tiled_w ||
1223 key->rt_samples != key->dst_samples ||
1224 key->rt_layout != key->dst_layout) {
1225 dst_pos = blorp_nir_encode_msaa(&b, dst_pos, key->rt_samples,
1226 key->rt_layout);
1227 /* Now (X, Y, S) = detile(rt_tiling, offset) */
1228 if (rt_tiled_w != key->dst_tiled_w)
1229 dst_pos = blorp_nir_retile_y_to_w(&b, dst_pos);
1230 /* Now (X, Y, S) = detile(rt_tiling, offset) */
1231 dst_pos = blorp_nir_decode_msaa(&b, dst_pos, key->dst_samples,
1232 key->dst_layout);
1233 }
1234
1235 nir_def *comp = NULL;
1236 if (key->dst_rgb) {
1237 /* The destination image is bound as a red texture three times as wide
1238 * as the actual image. Our shader is effectively running one color
1239 * component at a time. We need to save off the component and adjust
1240 * the destination position.
1241 */
1242 assert(dst_pos->num_components == 2);
1243 nir_def *dst_x = nir_channel(&b, dst_pos, 0);
1244 comp = nir_umod_imm(&b, dst_x, 3);
1245 dst_pos = nir_vec2(&b, nir_idiv(&b, dst_x, nir_imm_int(&b, 3)),
1246 nir_channel(&b, dst_pos, 1));
1247 }
1248
1249 /* Now (X, Y, S) = decode_msaa(dst_samples, detile(dst_tiling, offset)).
1250 *
1251 * That is: X, Y and S now contain the true coordinates and sample index of
1252 * the data that the WM thread should output.
1253 *
1254 * If we need to kill pixels that are outside the destination rectangle,
1255 * now is the time to do it.
1256 */
1257 nir_if *bounds_if = NULL;
1258 if (key->use_kill) {
1259 nir_def *bounds_rect = nir_load_var(&b, v.v_bounds_rect);
1260 nir_def *in_bounds = blorp_check_in_bounds(&b, bounds_rect,
1261 dst_pos);
1262 if (!compute)
1263 nir_discard_if(&b, nir_inot(&b, in_bounds));
1264 else
1265 bounds_if = nir_push_if(&b, in_bounds);
1266 }
1267
1268 src_pos = blorp_blit_apply_transform(&b, nir_i2f32(&b, dst_pos), &v);
1269 if (dst_pos->num_components == 3) {
1270 /* The sample coordinate is an integer that we want left alone but
1271 * blorp_blit_apply_transform() blindly applies the transform to all
1272 * three coordinates. Grab the original sample index.
1273 */
1274 src_pos = nir_vec3(&b, nir_channel(&b, src_pos, 0),
1275 nir_channel(&b, src_pos, 1),
1276 nir_channel(&b, dst_pos, 2));
1277 }
1278
1279 /* If the source image is not multisampled, then we want to fetch sample
1280 * number 0, because that's the only sample there is.
1281 */
1282 if (key->src_samples == 1)
1283 src_pos = nir_trim_vector(&b, src_pos, 2);
1284
1285 /* X, Y, and S are now the coordinates of the pixel in the source image
1286 * that we want to texture from. Exception: if we are blending, then S is
1287 * irrelevant, because we are going to fetch all samples.
1288 */
1289 switch (key->filter) {
1290 case BLORP_FILTER_NONE:
1291 case BLORP_FILTER_NEAREST:
1292 case BLORP_FILTER_SAMPLE_0:
1293 /* We're going to use texelFetch, so we need integers */
1294 if (src_pos->num_components == 2) {
1295 src_pos = nir_f2i32(&b, src_pos);
1296 } else {
1297 assert(src_pos->num_components == 3);
1298 src_pos = nir_vec3(&b, nir_channel(&b, nir_f2i32(&b, src_pos), 0),
1299 nir_channel(&b, nir_f2i32(&b, src_pos), 1),
1300 nir_channel(&b, src_pos, 2));
1301 }
1302
1303 /* We aren't blending, which means we just want to fetch a single
1304 * sample from the source surface. The address that we want to fetch
1305 * from is related to the X, Y and S values according to the formula:
1306 *
1307 * (X, Y, S) = decode_msaa(src_samples, detile(src_tiling, offset)).
1308 *
1309 * If the actual tiling and sample count of the source surface are
1310 * not the same as the configuration of the texture, then we need to
1311 * adjust the coordinates to compensate for the difference.
1312 */
1313 if (tex_tiled_w != key->src_tiled_w ||
1314 key->tex_samples != key->src_samples ||
1315 key->tex_layout != key->src_layout) {
1316 src_pos = blorp_nir_encode_msaa(&b, src_pos, key->src_samples,
1317 key->src_layout);
1318 /* Now (X, Y, S) = detile(src_tiling, offset) */
1319 if (tex_tiled_w != key->src_tiled_w)
1320 src_pos = blorp_nir_retile_w_to_y(&b, src_pos);
1321 /* Now (X, Y, S) = detile(tex_tiling, offset) */
1322 src_pos = blorp_nir_decode_msaa(&b, src_pos, key->tex_samples,
1323 key->tex_layout);
1324 }
1325
1326 if (key->need_src_offset)
1327 src_pos = nir_iadd(&b, src_pos, nir_load_var(&b, v.v_src_offset));
1328
1329 /* Now (X, Y, S) = decode_msaa(tex_samples, detile(tex_tiling, offset)).
1330 *
1331 * In other words: X, Y, and S now contain values which, when passed to
1332 * the texturing unit, will cause data to be read from the correct
1333 * memory location. So we can fetch the texel now.
1334 */
1335 if (key->src_samples == 1) {
1336 color = blorp_nir_txf(&b, &v, src_pos, key->texture_data_type);
1337 } else {
1338 nir_def *mcs = NULL;
1339 if (isl_aux_usage_has_mcs(key->tex_aux_usage))
1340 mcs = blorp_blit_txf_ms_mcs(&b, &v, src_pos);
1341
1342 color = blorp_nir_txf_ms(&b, &v, src_pos, mcs, key->texture_data_type);
1343 }
1344 break;
1345
1346 case BLORP_FILTER_BILINEAR:
1347 assert(!key->src_tiled_w);
1348 assert(key->tex_samples == key->src_samples);
1349 assert(key->tex_layout == key->src_layout);
1350
1351 if (key->src_samples == 1) {
1352 color = blorp_nir_tex(&b, &v, key, src_pos);
1353 } else {
1354 assert(!key->use_kill);
1355 color = blorp_nir_manual_blend_bilinear(&b, src_pos, key->src_samples,
1356 key, &v);
1357 }
1358 break;
1359
1360 case BLORP_FILTER_AVERAGE:
1361 case BLORP_FILTER_MIN_SAMPLE:
1362 case BLORP_FILTER_MAX_SAMPLE:
1363 assert(!key->src_tiled_w);
1364 assert(key->tex_samples == key->src_samples);
1365 assert(key->tex_layout == key->src_layout);
1366
1367 /* Resolves (effecively) use texelFetch, so we need integers and we
1368 * don't care about the sample index if we got one.
1369 */
1370 src_pos = nir_f2i32(&b, nir_trim_vector(&b, src_pos, 2));
1371
1372 if (devinfo->ver == 6) {
1373 /* Because gfx6 only supports 4x interleved MSAA, we can do all the
1374 * blending we need with a single linear-interpolated texture lookup
1375 * at the center of the sample. The texture coordinates to be odd
1376 * integers so that they correspond to the center of a 2x2 block
1377 * representing the four samples that maxe up a pixel. So we need
1378 * to multiply our X and Y coordinates each by 2 and then add 1.
1379 */
1380 assert(key->src_coords_normalized);
1381 assert(key->filter == BLORP_FILTER_AVERAGE);
1382 src_pos = nir_fadd_imm(&b,
1383 nir_i2f32(&b, src_pos),
1384 0.5f);
1385 color = blorp_nir_tex(&b, &v, key, src_pos);
1386 } else {
1387 /* Gfx7+ hardware doesn't automatically blend. */
1388 color = blorp_nir_combine_samples(&b, &v, src_pos, key->src_samples,
1389 key->tex_aux_usage,
1390 key->texture_data_type,
1391 key->filter);
1392 }
1393 break;
1394
1395 default:
1396 unreachable("Invalid blorp filter");
1397 }
1398
1399 if (!isl_swizzle_is_identity(key->src_swizzle)) {
1400 color = swizzle_color(&b, color, key->src_swizzle,
1401 key->texture_data_type);
1402 }
1403
1404 if (!isl_swizzle_is_identity(key->dst_swizzle)) {
1405 color = swizzle_color(&b, color, isl_swizzle_invert(key->dst_swizzle),
1406 nir_type_int);
1407 }
1408
1409 if (key->format_bit_cast) {
1410 assert(isl_swizzle_is_identity(key->src_swizzle));
1411 assert(isl_swizzle_is_identity(key->dst_swizzle));
1412 color = bit_cast_color(&b, color, key);
1413 } else if (key->dst_format) {
1414 color = convert_color(&b, color, key);
1415 } else if (key->uint32_to_sint) {
1416 /* Normally the hardware will take care of converting values from/to
1417 * the source and destination formats. But a few cases need help.
1418 *
1419 * The Skylake PRM, volume 07, page 658 has a programming note:
1420 *
1421 * "When using SINT or UINT rendertarget surface formats, Blending
1422 * must be DISABLED. The Pre-Blend Color Clamp Enable and Color
1423 * Clamp Range fields are ignored, and an implied clamp to the
1424 * rendertarget surface format is performed."
1425 *
1426 * For UINT to SINT blits, our sample operation gives us a uint32_t,
1427 * but our render target write expects a signed int32_t number. If we
1428 * simply passed the value along, the hardware would interpret a value
1429 * with bit 31 set as a negative value, clamping it to the largest
1430 * negative number the destination format could represent. But the
1431 * actual source value is a positive number, so we want to clamp it
1432 * to INT_MAX. To fix this, we explicitly take min(color, INT_MAX).
1433 */
1434 color = nir_umin(&b, color, nir_imm_int(&b, INT32_MAX));
1435 } else if (key->sint32_to_uint) {
1436 /* Similar to above, but clamping negative numbers to zero. */
1437 color = nir_imax(&b, color, nir_imm_int(&b, 0));
1438 }
1439
1440 if (key->dst_rgb) {
1441 /* The destination image is bound as a red texture three times as wide
1442 * as the actual image. Our shader is effectively running one color
1443 * component at a time. We need to pick off the appropriate component
1444 * from the source color and write that to destination red.
1445 */
1446 assert(dst_pos->num_components == 2);
1447
1448 nir_def *color_component =
1449 nir_bcsel(&b, nir_ieq_imm(&b, comp, 0),
1450 nir_channel(&b, color, 0),
1451 nir_bcsel(&b, nir_ieq_imm(&b, comp, 1),
1452 nir_channel(&b, color, 1),
1453 nir_channel(&b, color, 2)));
1454
1455 nir_def *u = nir_undef(&b, 1, 32);
1456 color = nir_vec4(&b, color_component, u, u, u);
1457 }
1458
1459 if (compute) {
1460 nir_def *store_pos = nir_load_global_invocation_id(&b, 32);
1461 nir_image_store(&b, nir_imm_int(&b, 0),
1462 nir_pad_vector_imm_int(&b, store_pos, 0, 4),
1463 nir_imm_int(&b, 0),
1464 nir_pad_vector_imm_int(&b, color, 0, 4),
1465 nir_imm_int(&b, 0),
1466 .image_dim = GLSL_SAMPLER_DIM_2D,
1467 .image_array = true,
1468 .access = ACCESS_NON_READABLE);
1469 } else if (key->dst_usage == ISL_SURF_USAGE_RENDER_TARGET_BIT) {
1470 nir_variable *color_out =
1471 nir_variable_create(b.shader, nir_var_shader_out,
1472 glsl_vec4_type(), "gl_FragColor");
1473 color_out->data.location = FRAG_RESULT_COLOR;
1474 nir_store_var(&b, color_out, color, 0xf);
1475 } else if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
1476 nir_variable *depth_out =
1477 nir_variable_create(b.shader, nir_var_shader_out,
1478 glsl_float_type(), "gl_FragDepth");
1479 depth_out->data.location = FRAG_RESULT_DEPTH;
1480 nir_store_var(&b, depth_out, nir_channel(&b, color, 0), 0x1);
1481 } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
1482 nir_variable *stencil_out =
1483 nir_variable_create(b.shader, nir_var_shader_out,
1484 glsl_int_type(), "gl_FragStencilRef");
1485 stencil_out->data.location = FRAG_RESULT_STENCIL;
1486 nir_store_var(&b, stencil_out, nir_channel(&b, color, 0), 0x1);
1487 } else {
1488 unreachable("Invalid destination usage");
1489 }
1490
1491 if (bounds_if)
1492 nir_pop_if(&b, bounds_if);
1493
1494 return b.shader;
1495 }
1496
1497 static bool
blorp_get_blit_kernel_fs(struct blorp_batch * batch,struct blorp_params * params,const struct blorp_blit_prog_key * key)1498 blorp_get_blit_kernel_fs(struct blorp_batch *batch,
1499 struct blorp_params *params,
1500 const struct blorp_blit_prog_key *key)
1501 {
1502 struct blorp_context *blorp = batch->blorp;
1503
1504 if (blorp->lookup_shader(batch, key, sizeof(*key),
1505 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data))
1506 return true;
1507
1508 void *mem_ctx = ralloc_context(NULL);
1509
1510 nir_shader *nir = blorp_build_nir_shader(blorp, batch, mem_ctx, key);
1511 nir->info.name =
1512 ralloc_strdup(nir, blorp_shader_type_to_name(key->base.shader_type));
1513
1514 const bool multisample_fbo = key->rt_samples > 1;
1515
1516 const struct blorp_program p =
1517 blorp_compile_fs(blorp, mem_ctx, nir, multisample_fbo, false);
1518
1519 bool result =
1520 blorp->upload_shader(batch, MESA_SHADER_FRAGMENT,
1521 key, sizeof(*key),
1522 p.kernel, p.kernel_size,
1523 p.prog_data, p.prog_data_size,
1524 ¶ms->wm_prog_kernel, ¶ms->wm_prog_data);
1525
1526 ralloc_free(mem_ctx);
1527 return result;
1528 }
1529
1530 static bool
blorp_get_blit_kernel_cs(struct blorp_batch * batch,struct blorp_params * params,const struct blorp_blit_prog_key * prog_key)1531 blorp_get_blit_kernel_cs(struct blorp_batch *batch,
1532 struct blorp_params *params,
1533 const struct blorp_blit_prog_key *prog_key)
1534 {
1535 struct blorp_context *blorp = batch->blorp;
1536
1537 if (blorp->lookup_shader(batch, prog_key, sizeof(*prog_key),
1538 ¶ms->cs_prog_kernel, ¶ms->cs_prog_data))
1539 return true;
1540
1541 void *mem_ctx = ralloc_context(NULL);
1542
1543 nir_shader *nir = blorp_build_nir_shader(blorp, batch, mem_ctx,
1544 prog_key);
1545 nir->info.name = ralloc_strdup(nir, "BLORP-gpgpu-blit");
1546 blorp_set_cs_dims(nir, prog_key->local_y);
1547
1548 assert(prog_key->rt_samples == 1);
1549
1550 const struct blorp_program p =
1551 blorp_compile_cs(blorp, mem_ctx, nir);
1552
1553 bool result =
1554 blorp->upload_shader(batch, MESA_SHADER_COMPUTE,
1555 prog_key, sizeof(*prog_key),
1556 p.kernel, p.kernel_size,
1557 p.prog_data, p.prog_data_size,
1558 ¶ms->cs_prog_kernel, ¶ms->cs_prog_data);
1559
1560 ralloc_free(mem_ctx);
1561 return result;
1562 }
1563
1564 static void
blorp_setup_coord_transform(struct blorp_coord_transform * xform,float src0,float src1,float dst0,float dst1,bool mirror)1565 blorp_setup_coord_transform(struct blorp_coord_transform *xform,
1566 float src0, float src1,
1567 float dst0, float dst1,
1568 bool mirror)
1569 {
1570 double scale = (double)(src1 - src0) / (double)(dst1 - dst0);
1571 if (!mirror) {
1572 /* When not mirroring a coordinate (say, X), we need:
1573 * src_x - src_x0 = (dst_x - dst_x0 + 0.5) * scale
1574 * Therefore:
1575 * src_x = src_x0 + (dst_x - dst_x0 + 0.5) * scale
1576 *
1577 * blorp program uses "round toward zero" to convert the
1578 * transformed floating point coordinates to integer coordinates,
1579 * whereas the behaviour we actually want is "round to nearest",
1580 * so 0.5 provides the necessary correction.
1581 */
1582 xform->multiplier = scale;
1583 xform->offset = src0 + (-(double)dst0 + 0.5) * scale;
1584 } else {
1585 /* When mirroring X we need:
1586 * src_x - src_x0 = dst_x1 - dst_x - 0.5
1587 * Therefore:
1588 * src_x = src_x0 + (dst_x1 -dst_x - 0.5) * scale
1589 */
1590 xform->multiplier = -scale;
1591 xform->offset = src0 + ((double)dst1 - 0.5) * scale;
1592 }
1593 }
1594
1595 static inline void
surf_get_intratile_offset_px(struct blorp_surface_info * info,uint32_t * tile_x_px,uint32_t * tile_y_px)1596 surf_get_intratile_offset_px(struct blorp_surface_info *info,
1597 uint32_t *tile_x_px, uint32_t *tile_y_px)
1598 {
1599 if (info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1600 struct isl_extent2d px_size_sa =
1601 isl_get_interleaved_msaa_px_size_sa(info->surf.samples);
1602 assert(info->tile_x_sa % px_size_sa.width == 0);
1603 assert(info->tile_y_sa % px_size_sa.height == 0);
1604 *tile_x_px = info->tile_x_sa / px_size_sa.width;
1605 *tile_y_px = info->tile_y_sa / px_size_sa.height;
1606 } else {
1607 *tile_x_px = info->tile_x_sa;
1608 *tile_y_px = info->tile_y_sa;
1609 }
1610 }
1611
1612 void
blorp_surf_convert_to_single_slice(const struct isl_device * isl_dev,struct blorp_surface_info * info)1613 blorp_surf_convert_to_single_slice(const struct isl_device *isl_dev,
1614 struct blorp_surface_info *info)
1615 {
1616 bool ok UNUSED;
1617
1618 /* It would be insane to try and do this on a compressed surface */
1619 assert(info->aux_usage == ISL_AUX_USAGE_NONE);
1620
1621 /* Just bail if we have nothing to do. */
1622 if (info->surf.dim == ISL_SURF_DIM_2D &&
1623 info->view.base_level == 0 && info->view.base_array_layer == 0 &&
1624 info->surf.levels == 1 && info->surf.logical_level0_px.array_len == 1)
1625 return;
1626
1627 /* If this gets triggered then we've gotten here twice which. This
1628 * shouldn't happen thanks to the above early return.
1629 */
1630 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
1631
1632 uint32_t layer = 0, z = 0;
1633 if (info->surf.dim == ISL_SURF_DIM_3D)
1634 z = info->view.base_array_layer + info->z_offset;
1635 else
1636 layer = info->view.base_array_layer;
1637
1638 uint64_t offset_B;
1639 isl_surf_get_image_surf(isl_dev, &info->surf,
1640 info->view.base_level, layer, z,
1641 &info->surf,
1642 &offset_B, &info->tile_x_sa, &info->tile_y_sa);
1643 info->addr.offset += offset_B;
1644
1645 uint32_t tile_x_px, tile_y_px;
1646 surf_get_intratile_offset_px(info, &tile_x_px, &tile_y_px);
1647
1648 /* Instead of using the X/Y Offset fields in RENDER_SURFACE_STATE, we place
1649 * the image at the tile boundary and offset our sampling or rendering.
1650 * For this reason, we need to grow the image by the offset to ensure that
1651 * the hardware doesn't think we've gone past the edge.
1652 */
1653 info->surf.logical_level0_px.w += tile_x_px;
1654 info->surf.logical_level0_px.h += tile_y_px;
1655 info->surf.phys_level0_sa.w += info->tile_x_sa;
1656 info->surf.phys_level0_sa.h += info->tile_y_sa;
1657
1658 /* The view is also different now. */
1659 info->view.base_level = 0;
1660 info->view.levels = 1;
1661 info->view.base_array_layer = 0;
1662 info->view.array_len = 1;
1663 info->z_offset = 0;
1664 }
1665
1666 void
blorp_surf_fake_interleaved_msaa(const struct isl_device * isl_dev,struct blorp_surface_info * info)1667 blorp_surf_fake_interleaved_msaa(const struct isl_device *isl_dev,
1668 struct blorp_surface_info *info)
1669 {
1670 assert(info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED);
1671
1672 /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1673 blorp_surf_convert_to_single_slice(isl_dev, info);
1674
1675 info->surf.logical_level0_px = info->surf.phys_level0_sa;
1676 info->surf.samples = 1;
1677 info->surf.msaa_layout = ISL_MSAA_LAYOUT_NONE;
1678 }
1679
1680 void
blorp_surf_retile_w_to_y(const struct isl_device * isl_dev,struct blorp_surface_info * info)1681 blorp_surf_retile_w_to_y(const struct isl_device *isl_dev,
1682 struct blorp_surface_info *info)
1683 {
1684 assert(info->surf.tiling == ISL_TILING_W);
1685
1686 /* First, we need to convert it to a simple 1-level 1-layer 2-D surface */
1687 blorp_surf_convert_to_single_slice(isl_dev, info);
1688
1689 /* On gfx7+, we don't have interleaved multisampling for color render
1690 * targets so we have to fake it.
1691 *
1692 * TODO: Are we sure we don't also need to fake it on gfx6?
1693 */
1694 if (isl_dev->info->ver > 6 &&
1695 info->surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1696 blorp_surf_fake_interleaved_msaa(isl_dev, info);
1697 }
1698
1699 if (isl_dev->info->ver == 6 || isl_dev->info->ver == 7) {
1700 /* Gfx6-7 stencil buffers have a very large alignment coming in from the
1701 * miptree. It's out-of-bounds for what the surface state can handle.
1702 * Since we have a single layer and level, it doesn't really matter as
1703 * long as we don't pass a bogus value into isl_surf_fill_state().
1704 */
1705 info->surf.image_alignment_el = isl_extent3d(4, 2, 1);
1706 }
1707
1708 /* Now that we've converted everything to a simple 2-D surface with only
1709 * one miplevel, we can go about retiling it.
1710 */
1711 const unsigned x_align = 8, y_align = info->surf.samples != 0 ? 8 : 4;
1712 info->surf.tiling = ISL_TILING_Y0;
1713 info->surf.logical_level0_px.width =
1714 ALIGN(info->surf.logical_level0_px.width, x_align) * 2;
1715 info->surf.logical_level0_px.height =
1716 ALIGN(info->surf.logical_level0_px.height, y_align) / 2;
1717 info->tile_x_sa *= 2;
1718 info->tile_y_sa /= 2;
1719 }
1720
1721 static bool
can_shrink_surface(const struct blorp_surface_info * surf)1722 can_shrink_surface(const struct blorp_surface_info *surf)
1723 {
1724 /* The current code doesn't support offsets into the aux buffers. This
1725 * should be possible, but we need to make sure the offset is page
1726 * aligned for both the surface and the aux buffer surface. Generally
1727 * this mean using the page aligned offset for the aux buffer.
1728 *
1729 * Currently the cases where we must split the blit are limited to cases
1730 * where we don't have a aux buffer.
1731 */
1732 if (surf->aux_addr.buffer != NULL)
1733 return false;
1734
1735 /* We can't support splitting the blit for gen <= 7, because the qpitch
1736 * size is calculated by the hardware based on the surface height for
1737 * gen <= 7. In gen >= 8, the qpitch is controlled by the driver.
1738 */
1739 if (surf->surf.msaa_layout == ISL_MSAA_LAYOUT_ARRAY)
1740 return false;
1741
1742 return true;
1743 }
1744
1745 static unsigned
get_max_surface_size(const struct intel_device_info * devinfo,const struct blorp_surface_info * surf)1746 get_max_surface_size(const struct intel_device_info *devinfo,
1747 const struct blorp_surface_info *surf)
1748 {
1749 const unsigned max = devinfo->ver >= 7 ? 16384 : 8192;
1750 if (split_blorp_blit_debug && can_shrink_surface(surf))
1751 return max >> 4; /* A smaller restriction when debug is enabled */
1752 else
1753 return max;
1754 }
1755
1756 struct blt_axis {
1757 double src0, src1, dst0, dst1;
1758 bool mirror;
1759 };
1760
1761 struct blt_coords {
1762 struct blt_axis x, y;
1763 };
1764
1765 static enum isl_format
get_red_format_for_rgb_format(enum isl_format format)1766 get_red_format_for_rgb_format(enum isl_format format)
1767 {
1768 const struct isl_format_layout *fmtl = isl_format_get_layout(format);
1769
1770 switch (fmtl->channels.r.bits) {
1771 case 8:
1772 switch (fmtl->channels.r.type) {
1773 case ISL_UNORM:
1774 return ISL_FORMAT_R8_UNORM;
1775 case ISL_SNORM:
1776 return ISL_FORMAT_R8_SNORM;
1777 case ISL_UINT:
1778 return ISL_FORMAT_R8_UINT;
1779 case ISL_SINT:
1780 return ISL_FORMAT_R8_SINT;
1781 default:
1782 unreachable("Invalid 8-bit RGB channel type");
1783 }
1784 case 16:
1785 switch (fmtl->channels.r.type) {
1786 case ISL_UNORM:
1787 return ISL_FORMAT_R16_UNORM;
1788 case ISL_SNORM:
1789 return ISL_FORMAT_R16_SNORM;
1790 case ISL_SFLOAT:
1791 return ISL_FORMAT_R16_FLOAT;
1792 case ISL_UINT:
1793 return ISL_FORMAT_R16_UINT;
1794 case ISL_SINT:
1795 return ISL_FORMAT_R16_SINT;
1796 default:
1797 unreachable("Invalid 8-bit RGB channel type");
1798 }
1799 case 32:
1800 switch (fmtl->channels.r.type) {
1801 case ISL_SFLOAT:
1802 return ISL_FORMAT_R32_FLOAT;
1803 case ISL_UINT:
1804 return ISL_FORMAT_R32_UINT;
1805 case ISL_SINT:
1806 return ISL_FORMAT_R32_SINT;
1807 default:
1808 unreachable("Invalid 8-bit RGB channel type");
1809 }
1810 default:
1811 unreachable("Invalid number of red channel bits");
1812 }
1813 }
1814
1815 void
surf_fake_rgb_with_red(const struct isl_device * isl_dev,struct blorp_surface_info * info)1816 surf_fake_rgb_with_red(const struct isl_device *isl_dev,
1817 struct blorp_surface_info *info)
1818 {
1819 blorp_surf_convert_to_single_slice(isl_dev, info);
1820
1821 info->surf.logical_level0_px.width *= 3;
1822 info->surf.phys_level0_sa.width *= 3;
1823 info->tile_x_sa *= 3;
1824
1825 enum isl_format red_format =
1826 get_red_format_for_rgb_format(info->view.format);
1827
1828 assert(isl_format_get_layout(red_format)->channels.r.type ==
1829 isl_format_get_layout(info->view.format)->channels.r.type);
1830 assert(isl_format_get_layout(red_format)->channels.r.bits ==
1831 isl_format_get_layout(info->view.format)->channels.r.bits);
1832
1833 info->surf.format = info->view.format = red_format;
1834
1835 if (isl_dev->info->verx10 >= 125) {
1836 /* The horizontal alignment is in units of texels for NPOT formats, and
1837 * bytes for other formats. Since the only allowed alignment units are
1838 * powers of two, there's no way to convert the alignment.
1839 *
1840 * Thankfully, the value doesn't matter since we're only a single slice.
1841 * Pick one allowed by isl_gfx125_choose_image_alignment_el.
1842 */
1843 info->surf.image_alignment_el.w =
1844 128 / (isl_format_get_layout(red_format)->bpb / 8);
1845 }
1846 }
1847
1848 enum blit_shrink_status {
1849 BLIT_NO_SHRINK = 0,
1850 BLIT_SRC_WIDTH_SHRINK = (1 << 0),
1851 BLIT_DST_WIDTH_SHRINK = (1 << 1),
1852 BLIT_SRC_HEIGHT_SHRINK = (1 << 2),
1853 BLIT_DST_HEIGHT_SHRINK = (1 << 3),
1854 };
1855
1856 /* Try to blit. If the surface parameters exceed the size allowed by hardware,
1857 * then enum blit_shrink_status will be returned. If BLIT_NO_SHRINK is
1858 * returned, then the blit was successful.
1859 */
1860 static enum blit_shrink_status
try_blorp_blit(struct blorp_batch * batch,struct blorp_params * params,struct blorp_blit_prog_key * key,struct blt_coords * coords)1861 try_blorp_blit(struct blorp_batch *batch,
1862 struct blorp_params *params,
1863 struct blorp_blit_prog_key *key,
1864 struct blt_coords *coords)
1865 {
1866 const struct intel_device_info *devinfo = batch->blorp->isl_dev->info;
1867
1868 if (params->dst.surf.usage & ISL_SURF_USAGE_DEPTH_BIT) {
1869 if (devinfo->ver >= 7) {
1870 /* We can render as depth on Gfx5 but there's no real advantage since
1871 * it doesn't support MSAA or HiZ. On Gfx4, we can't always render
1872 * to depth due to issues with depth buffers and mip-mapping. On
1873 * Gfx6, we can do everything but we have weird offsetting for HiZ
1874 * and stencil. It's easier to just render using the color pipe
1875 * on those platforms.
1876 */
1877 key->dst_usage = ISL_SURF_USAGE_DEPTH_BIT;
1878 } else {
1879 key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1880 }
1881 } else if (params->dst.surf.usage & ISL_SURF_USAGE_STENCIL_BIT) {
1882 assert(params->dst.surf.format == ISL_FORMAT_R8_UINT);
1883 if (devinfo->ver >= 9 && !(batch->flags & BLORP_BATCH_USE_COMPUTE)) {
1884 key->dst_usage = ISL_SURF_USAGE_STENCIL_BIT;
1885 } else {
1886 key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1887 }
1888 } else {
1889 key->dst_usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1890 }
1891
1892 if (isl_format_has_sint_channel(params->src.view.format)) {
1893 key->texture_data_type = nir_type_int;
1894 } else if (isl_format_has_uint_channel(params->src.view.format)) {
1895 key->texture_data_type = nir_type_uint;
1896 } else {
1897 key->texture_data_type = nir_type_float;
1898 }
1899
1900 /* src_samples and dst_samples are the true sample counts */
1901 key->src_samples = params->src.surf.samples;
1902 key->dst_samples = params->dst.surf.samples;
1903
1904 key->tex_aux_usage = params->src.aux_usage;
1905
1906 /* src_layout and dst_layout indicate the true MSAA layout used by src and
1907 * dst.
1908 */
1909 key->src_layout = params->src.surf.msaa_layout;
1910 key->dst_layout = params->dst.surf.msaa_layout;
1911
1912 /* Round floating point values to nearest integer to avoid "off by one texel"
1913 * kind of errors when blitting.
1914 */
1915 params->x0 = params->wm_inputs.bounds_rect.x0 = round(coords->x.dst0);
1916 params->y0 = params->wm_inputs.bounds_rect.y0 = round(coords->y.dst0);
1917 params->x1 = params->wm_inputs.bounds_rect.x1 = round(coords->x.dst1);
1918 params->y1 = params->wm_inputs.bounds_rect.y1 = round(coords->y.dst1);
1919
1920 blorp_setup_coord_transform(¶ms->wm_inputs.coord_transform[0],
1921 coords->x.src0, coords->x.src1,
1922 coords->x.dst0, coords->x.dst1,
1923 coords->x.mirror);
1924 blorp_setup_coord_transform(¶ms->wm_inputs.coord_transform[1],
1925 coords->y.src0, coords->y.src1,
1926 coords->y.dst0, coords->y.dst1,
1927 coords->y.mirror);
1928
1929
1930 if (devinfo->ver == 4) {
1931 /* The MinLOD and MinimumArrayElement don't work properly for cube maps.
1932 * Convert them to a single slice on gfx4.
1933 */
1934 if (params->dst.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1935 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->dst);
1936 key->need_dst_offset = true;
1937 }
1938
1939 if (params->src.surf.usage & ISL_SURF_USAGE_CUBE_BIT) {
1940 blorp_surf_convert_to_single_slice(batch->blorp->isl_dev, ¶ms->src);
1941 key->need_src_offset = true;
1942 }
1943 }
1944
1945 if (devinfo->ver > 6 &&
1946 !isl_surf_usage_is_depth_or_stencil(key->dst_usage) &&
1947 params->dst.surf.msaa_layout == ISL_MSAA_LAYOUT_INTERLEAVED) {
1948 assert(params->dst.surf.samples > 1);
1949
1950 /* We must expand the rectangle we send through the rendering pipeline,
1951 * to account for the fact that we are mapping the destination region as
1952 * single-sampled when it is in fact multisampled. We must also align
1953 * it to a multiple of the multisampling pattern, because the
1954 * differences between multisampled and single-sampled surface formats
1955 * will mean that pixels are scrambled within the multisampling pattern.
1956 * TODO: what if this makes the coordinates too large?
1957 *
1958 * Note: this only works if the destination surface uses the IMS layout.
1959 * If it's UMS, then we have no choice but to set up the rendering
1960 * pipeline as multisampled.
1961 */
1962 struct isl_extent2d px_size_sa =
1963 isl_get_interleaved_msaa_px_size_sa(params->dst.surf.samples);
1964 params->x0 = ROUND_DOWN_TO(params->x0, 2) * px_size_sa.width;
1965 params->y0 = ROUND_DOWN_TO(params->y0, 2) * px_size_sa.height;
1966 params->x1 = ALIGN(params->x1, 2) * px_size_sa.width;
1967 params->y1 = ALIGN(params->y1, 2) * px_size_sa.height;
1968
1969 blorp_surf_fake_interleaved_msaa(batch->blorp->isl_dev, ¶ms->dst);
1970
1971 key->use_kill = true;
1972 key->need_dst_offset = true;
1973 }
1974
1975 if (params->dst.surf.tiling == ISL_TILING_W &&
1976 key->dst_usage != ISL_SURF_USAGE_STENCIL_BIT) {
1977 /* We must modify the rectangle we send through the rendering pipeline
1978 * (and the size and x/y offset of the destination surface), to account
1979 * for the fact that we are mapping it as Y-tiled when it is in fact
1980 * W-tiled.
1981 *
1982 * Both Y tiling and W tiling can be understood as organizations of
1983 * 32-byte sub-tiles; within each 32-byte sub-tile, the layout of pixels
1984 * is different, but the layout of the 32-byte sub-tiles within the 4k
1985 * tile is the same (8 sub-tiles across by 16 sub-tiles down, in
1986 * column-major order). In Y tiling, the sub-tiles are 16 bytes wide
1987 * and 2 rows high; in W tiling, they are 8 bytes wide and 4 rows high.
1988 *
1989 * Therefore, to account for the layout differences within the 32-byte
1990 * sub-tiles, we must expand the rectangle so the X coordinates of its
1991 * edges are multiples of 8 (the W sub-tile width), and its Y
1992 * coordinates of its edges are multiples of 4 (the W sub-tile height).
1993 * Then we need to scale the X and Y coordinates of the rectangle to
1994 * account for the differences in aspect ratio between the Y and W
1995 * sub-tiles. We need to modify the layer width and height similarly.
1996 *
1997 * A correction needs to be applied when MSAA is in use: since
1998 * INTEL_MSAA_LAYOUT_IMS uses an interleaving pattern whose height is 4,
1999 * we need to align the Y coordinates to multiples of 8, so that when
2000 * they are divided by two they are still multiples of 4.
2001 *
2002 * Note: Since the x/y offset of the surface will be applied using the
2003 * SURFACE_STATE command packet, it will be invisible to the swizzling
2004 * code in the shader; therefore it needs to be in a multiple of the
2005 * 32-byte sub-tile size. Fortunately it is, since the sub-tile is 8
2006 * pixels wide and 4 pixels high (when viewed as a W-tiled stencil
2007 * buffer), and the miplevel alignment used for stencil buffers is 8
2008 * pixels horizontally and either 4 or 8 pixels vertically (see
2009 * intel_horizontal_texture_alignment_unit() and
2010 * intel_vertical_texture_alignment_unit()).
2011 *
2012 * Note: Also, since the SURFACE_STATE command packet can only apply
2013 * offsets that are multiples of 4 pixels horizontally and 2 pixels
2014 * vertically, it is important that the offsets will be multiples of
2015 * these sizes after they are converted into Y-tiled coordinates.
2016 * Fortunately they will be, since we know from above that the offsets
2017 * are a multiple of the 32-byte sub-tile size, and in Y-tiled
2018 * coordinates the sub-tile is 16 pixels wide and 2 pixels high.
2019 *
2020 * TODO: what if this makes the coordinates (or the texture size) too
2021 * large?
2022 */
2023 const unsigned x_align = 8;
2024 const unsigned y_align = params->dst.surf.samples != 0 ? 8 : 4;
2025 params->x0 = ROUND_DOWN_TO(params->x0, x_align) * 2;
2026 params->y0 = ROUND_DOWN_TO(params->y0, y_align) / 2;
2027 params->x1 = ALIGN(params->x1, x_align) * 2;
2028 params->y1 = ALIGN(params->y1, y_align) / 2;
2029
2030 /* Retile the surface to Y-tiled */
2031 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms->dst);
2032
2033 key->dst_tiled_w = true;
2034 key->use_kill = true;
2035 key->need_dst_offset = true;
2036
2037 if (params->dst.surf.samples > 1) {
2038 /* If the destination surface is a W-tiled multisampled stencil
2039 * buffer that we're mapping as Y tiled, then we need to arrange for
2040 * the WM program to run once per sample rather than once per pixel,
2041 * because the memory layout of related samples doesn't match between
2042 * W and Y tiling.
2043 */
2044 key->persample_msaa_dispatch = true;
2045 }
2046 }
2047
2048 if (devinfo->ver < 8 && params->src.surf.tiling == ISL_TILING_W) {
2049 /* On Haswell and earlier, we have to fake W-tiled sources as Y-tiled.
2050 * Broadwell adds support for sampling from stencil.
2051 *
2052 * See the comments above concerning x/y offset alignment for the
2053 * destination surface.
2054 *
2055 * TODO: what if this makes the texture size too large?
2056 */
2057 blorp_surf_retile_w_to_y(batch->blorp->isl_dev, ¶ms->src);
2058
2059 key->src_tiled_w = true;
2060 key->need_src_offset = true;
2061 }
2062
2063 /* tex_samples and rt_samples are the sample counts that are set up in
2064 * SURFACE_STATE.
2065 */
2066 key->tex_samples = params->src.surf.samples;
2067 key->rt_samples = params->dst.surf.samples;
2068
2069 /* tex_layout and rt_layout indicate the MSAA layout the GPU pipeline will
2070 * use to access the source and destination surfaces.
2071 */
2072 key->tex_layout = params->src.surf.msaa_layout;
2073 key->rt_layout = params->dst.surf.msaa_layout;
2074
2075 if (params->src.surf.samples > 0 && params->dst.surf.samples > 1) {
2076 /* We are blitting from a multisample buffer to a multisample buffer, so
2077 * we must preserve samples within a pixel. This means we have to
2078 * arrange for the WM program to run once per sample rather than once
2079 * per pixel.
2080 */
2081 key->persample_msaa_dispatch = true;
2082 }
2083
2084 params->num_samples = params->dst.surf.samples;
2085
2086 if ((key->filter == BLORP_FILTER_AVERAGE ||
2087 key->filter == BLORP_FILTER_BILINEAR) &&
2088 batch->blorp->isl_dev->info->ver <= 6) {
2089 /* Gfx4-5 don't support non-normalized texture coordinates */
2090 key->src_coords_normalized = true;
2091 params->wm_inputs.src_inv_size[0] =
2092 1.0f / u_minify(params->src.surf.logical_level0_px.width,
2093 params->src.view.base_level);
2094 params->wm_inputs.src_inv_size[1] =
2095 1.0f / u_minify(params->src.surf.logical_level0_px.height,
2096 params->src.view.base_level);
2097 }
2098
2099 if (isl_format_get_layout(params->dst.view.format)->bpb % 3 == 0) {
2100 /* We can't render to RGB formats natively because they aren't a
2101 * power-of-two size. Instead, we fake them by using a red format
2102 * with the same channel type and size and emitting shader code to
2103 * only write one channel at a time.
2104 */
2105 params->x0 *= 3;
2106 params->x1 *= 3;
2107
2108 /* If it happens to be sRGB, we need to force a conversion */
2109 if (params->dst.view.format == ISL_FORMAT_R8G8B8_UNORM_SRGB)
2110 key->dst_format = ISL_FORMAT_R8G8B8_UNORM_SRGB;
2111
2112 surf_fake_rgb_with_red(batch->blorp->isl_dev, ¶ms->dst);
2113
2114 key->dst_rgb = true;
2115 key->need_dst_offset = true;
2116 } else if (isl_format_is_rgbx(params->dst.view.format)) {
2117 /* We can handle RGBX formats easily enough by treating them as RGBA */
2118 params->dst.view.format =
2119 isl_format_rgbx_to_rgba(params->dst.view.format);
2120 } else if (params->dst.view.format == ISL_FORMAT_R24_UNORM_X8_TYPELESS &&
2121 key->dst_usage != ISL_SURF_USAGE_DEPTH_BIT) {
2122 key->dst_format = params->dst.view.format;
2123 params->dst.view.format = ISL_FORMAT_R32_UINT;
2124 } else if (params->dst.view.format == ISL_FORMAT_A4B4G4R4_UNORM) {
2125 params->dst.view.swizzle =
2126 isl_swizzle_compose(params->dst.view.swizzle,
2127 ISL_SWIZZLE(ALPHA, RED, GREEN, BLUE));
2128 params->dst.view.format = ISL_FORMAT_B4G4R4A4_UNORM;
2129 } else if (params->dst.view.format == ISL_FORMAT_L8_UNORM_SRGB) {
2130 key->dst_format = params->dst.view.format;
2131 params->dst.view.format = ISL_FORMAT_R8_UNORM;
2132 } else if (params->dst.view.format == ISL_FORMAT_R9G9B9E5_SHAREDEXP) {
2133 key->dst_format = params->dst.view.format;
2134 params->dst.view.format = ISL_FORMAT_R32_UINT;
2135 }
2136
2137 if (devinfo->verx10 <= 70 &&
2138 !isl_swizzle_is_identity(params->src.view.swizzle)) {
2139 key->src_swizzle = params->src.view.swizzle;
2140 params->src.view.swizzle = ISL_SWIZZLE_IDENTITY;
2141 } else {
2142 key->src_swizzle = ISL_SWIZZLE_IDENTITY;
2143 }
2144
2145 if (!isl_swizzle_supports_rendering(devinfo, params->dst.view.swizzle)) {
2146 key->dst_swizzle = params->dst.view.swizzle;
2147 params->dst.view.swizzle = ISL_SWIZZLE_IDENTITY;
2148 } else {
2149 key->dst_swizzle = ISL_SWIZZLE_IDENTITY;
2150 }
2151
2152 if (params->src.tile_x_sa || params->src.tile_y_sa) {
2153 assert(key->need_src_offset);
2154 surf_get_intratile_offset_px(¶ms->src,
2155 ¶ms->wm_inputs.src_offset.x,
2156 ¶ms->wm_inputs.src_offset.y);
2157 }
2158
2159 if (params->dst.tile_x_sa || params->dst.tile_y_sa) {
2160 assert(key->need_dst_offset);
2161 surf_get_intratile_offset_px(¶ms->dst,
2162 ¶ms->wm_inputs.dst_offset.x,
2163 ¶ms->wm_inputs.dst_offset.y);
2164 params->x0 += params->wm_inputs.dst_offset.x;
2165 params->y0 += params->wm_inputs.dst_offset.y;
2166 params->x1 += params->wm_inputs.dst_offset.x;
2167 params->y1 += params->wm_inputs.dst_offset.y;
2168 }
2169
2170 /* For some texture types, we need to pass the layer through the sampler. */
2171 params->wm_inputs.src_z = params->src.z_offset;
2172
2173 const bool compute =
2174 key->base.shader_pipeline == BLORP_SHADER_PIPELINE_COMPUTE;
2175 if (compute) {
2176 key->local_y = blorp_get_cs_local_y(params);
2177
2178 unsigned workgroup_width = 16 / key->local_y;
2179 unsigned workgroup_height = key->local_y;
2180
2181 /* If the rectangle being drawn isn't an exact multiple of the
2182 * workgroup size, we'll get extra invocations that should not
2183 * perform blits. We need to set use_kill to bounds check and
2184 * prevent those invocations from blitting.
2185 */
2186 if ((params->x0 % workgroup_width) != 0 ||
2187 (params->x1 % workgroup_width) != 0 ||
2188 (params->y0 % workgroup_height) != 0 ||
2189 (params->y1 % workgroup_height) != 0)
2190 key->use_kill = true;
2191 }
2192
2193 if (compute) {
2194 if (!blorp_get_blit_kernel_cs(batch, params, key))
2195 return 0;
2196 } else {
2197 if (!blorp_get_blit_kernel_fs(batch, params, key))
2198 return 0;
2199
2200 if (!blorp_ensure_sf_program(batch, params))
2201 return 0;
2202 }
2203
2204 unsigned result = 0;
2205 unsigned max_src_surface_size = get_max_surface_size(devinfo, ¶ms->src);
2206 if (params->src.surf.logical_level0_px.width > max_src_surface_size)
2207 result |= BLIT_SRC_WIDTH_SHRINK;
2208 if (params->src.surf.logical_level0_px.height > max_src_surface_size)
2209 result |= BLIT_SRC_HEIGHT_SHRINK;
2210
2211 unsigned max_dst_surface_size = get_max_surface_size(devinfo, ¶ms->dst);
2212 if (params->dst.surf.logical_level0_px.width > max_dst_surface_size)
2213 result |= BLIT_DST_WIDTH_SHRINK;
2214 if (params->dst.surf.logical_level0_px.height > max_dst_surface_size)
2215 result |= BLIT_DST_HEIGHT_SHRINK;
2216
2217 if (result == 0) {
2218 if (key->dst_usage == ISL_SURF_USAGE_DEPTH_BIT) {
2219 params->depth = params->dst;
2220 memset(¶ms->dst, 0, sizeof(params->dst));
2221 } else if (key->dst_usage == ISL_SURF_USAGE_STENCIL_BIT) {
2222 params->stencil = params->dst;
2223 params->stencil_mask = 0xff;
2224 memset(¶ms->dst, 0, sizeof(params->dst));
2225 }
2226
2227 batch->blorp->exec(batch, params);
2228 }
2229
2230 return result;
2231 }
2232
2233 /* Adjust split blit source coordinates for the current destination
2234 * coordinates.
2235 */
2236 static void
adjust_split_source_coords(const struct blt_axis * orig,struct blt_axis * split_coords,double scale)2237 adjust_split_source_coords(const struct blt_axis *orig,
2238 struct blt_axis *split_coords,
2239 double scale)
2240 {
2241 /* When scale is greater than 0, then we are growing from the start, so
2242 * src0 uses delta0, and src1 uses delta1. When scale is less than 0, the
2243 * source range shrinks from the end. In that case src0 is adjusted by
2244 * delta1, and src1 is adjusted by delta0.
2245 */
2246 double delta0 = scale * (split_coords->dst0 - orig->dst0);
2247 double delta1 = scale * (split_coords->dst1 - orig->dst1);
2248 split_coords->src0 = orig->src0 + (scale >= 0.0 ? delta0 : delta1);
2249 split_coords->src1 = orig->src1 + (scale >= 0.0 ? delta1 : delta0);
2250 }
2251
2252 static struct isl_extent2d
get_px_size_sa(const struct isl_surf * surf)2253 get_px_size_sa(const struct isl_surf *surf)
2254 {
2255 static const struct isl_extent2d one_to_one = { .w = 1, .h = 1 };
2256
2257 if (surf->msaa_layout != ISL_MSAA_LAYOUT_INTERLEAVED)
2258 return one_to_one;
2259 else
2260 return isl_get_interleaved_msaa_px_size_sa(surf->samples);
2261 }
2262
2263 static void
shrink_surface_params(const struct isl_device * dev,struct blorp_surface_info * info,double * x0,double * x1,double * y0,double * y1)2264 shrink_surface_params(const struct isl_device *dev,
2265 struct blorp_surface_info *info,
2266 double *x0, double *x1, double *y0, double *y1)
2267 {
2268 uint64_t offset_B;
2269 uint32_t x_offset_sa, y_offset_sa, size;
2270 struct isl_extent2d px_size_sa;
2271 int adjust;
2272
2273 blorp_surf_convert_to_single_slice(dev, info);
2274
2275 px_size_sa = get_px_size_sa(&info->surf);
2276
2277 /* Because this gets called after we lower compressed images, the tile
2278 * offsets may be non-zero and we need to incorporate them in our
2279 * calculations.
2280 */
2281 x_offset_sa = (uint32_t)*x0 * px_size_sa.w + info->tile_x_sa;
2282 y_offset_sa = (uint32_t)*y0 * px_size_sa.h + info->tile_y_sa;
2283 uint32_t tile_z_sa, tile_a;
2284 isl_tiling_get_intratile_offset_sa(info->surf.tiling, info->surf.dim,
2285 info->surf.msaa_layout,
2286 info->surf.format, info->surf.samples,
2287 info->surf.row_pitch_B,
2288 info->surf.array_pitch_el_rows,
2289 x_offset_sa, y_offset_sa, 0, 0,
2290 &offset_B,
2291 &info->tile_x_sa, &info->tile_y_sa,
2292 &tile_z_sa, &tile_a);
2293 assert(tile_z_sa == 0 && tile_a == 0);
2294
2295 info->addr.offset += offset_B;
2296
2297 adjust = (int)info->tile_x_sa / px_size_sa.w - (int)*x0;
2298 *x0 += adjust;
2299 *x1 += adjust;
2300 info->tile_x_sa = 0;
2301
2302 adjust = (int)info->tile_y_sa / px_size_sa.h - (int)*y0;
2303 *y0 += adjust;
2304 *y1 += adjust;
2305 info->tile_y_sa = 0;
2306
2307 size = MIN2((uint32_t)ceil(*x1), info->surf.logical_level0_px.width);
2308 info->surf.logical_level0_px.width = size;
2309 info->surf.phys_level0_sa.width = size * px_size_sa.w;
2310
2311 size = MIN2((uint32_t)ceil(*y1), info->surf.logical_level0_px.height);
2312 info->surf.logical_level0_px.height = size;
2313 info->surf.phys_level0_sa.height = size * px_size_sa.h;
2314 }
2315
2316 static void
do_blorp_blit(struct blorp_batch * batch,const struct blorp_params * orig_params,struct blorp_blit_prog_key * key,const struct blt_coords * orig)2317 do_blorp_blit(struct blorp_batch *batch,
2318 const struct blorp_params *orig_params,
2319 struct blorp_blit_prog_key *key,
2320 const struct blt_coords *orig)
2321 {
2322 struct blorp_params params;
2323 struct blt_coords blit_coords;
2324 struct blt_coords split_coords = *orig;
2325 double w = orig->x.dst1 - orig->x.dst0;
2326 double h = orig->y.dst1 - orig->y.dst0;
2327 double x_scale = (orig->x.src1 - orig->x.src0) / w;
2328 double y_scale = (orig->y.src1 - orig->y.src0) / h;
2329 if (orig->x.mirror)
2330 x_scale = -x_scale;
2331 if (orig->y.mirror)
2332 y_scale = -y_scale;
2333
2334 enum blit_shrink_status shrink = BLIT_NO_SHRINK;
2335 if (split_blorp_blit_debug) {
2336 if (can_shrink_surface(&orig_params->src))
2337 shrink |= BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK;
2338 if (can_shrink_surface(&orig_params->dst))
2339 shrink |= BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK;
2340 }
2341
2342 bool x_done, y_done;
2343 do {
2344 params = *orig_params;
2345 blit_coords = split_coords;
2346
2347 if (shrink & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK)) {
2348 shrink_surface_params(batch->blorp->isl_dev, ¶ms.src,
2349 &blit_coords.x.src0, &blit_coords.x.src1,
2350 &blit_coords.y.src0, &blit_coords.y.src1);
2351 key->need_src_offset = false;
2352 }
2353
2354 if (shrink & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2355 shrink_surface_params(batch->blorp->isl_dev, ¶ms.dst,
2356 &blit_coords.x.dst0, &blit_coords.x.dst1,
2357 &blit_coords.y.dst0, &blit_coords.y.dst1);
2358 key->need_dst_offset = false;
2359 }
2360
2361 enum blit_shrink_status result =
2362 try_blorp_blit(batch, ¶ms, key, &blit_coords);
2363
2364 if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_SRC_HEIGHT_SHRINK))
2365 assert(can_shrink_surface(&orig_params->src));
2366
2367 if (result & (BLIT_DST_WIDTH_SHRINK | BLIT_DST_HEIGHT_SHRINK))
2368 assert(can_shrink_surface(&orig_params->dst));
2369
2370 if (result & (BLIT_SRC_WIDTH_SHRINK | BLIT_DST_WIDTH_SHRINK)) {
2371 w /= 2.0;
2372 assert(w >= 1.0);
2373 split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2374 adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2375 }
2376 if (result & (BLIT_SRC_HEIGHT_SHRINK | BLIT_DST_HEIGHT_SHRINK)) {
2377 h /= 2.0;
2378 assert(h >= 1.0);
2379 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2380 adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2381 }
2382
2383 if (result) {
2384 /* We may get less bits set on result than we had already, so make
2385 * sure we remember all the ways in which a resize is required.
2386 */
2387 shrink |= result;
2388 continue;
2389 }
2390
2391 y_done = (orig->y.dst1 - split_coords.y.dst1 < 0.5);
2392 x_done = y_done && (orig->x.dst1 - split_coords.x.dst1 < 0.5);
2393 if (x_done) {
2394 break;
2395 } else if (y_done) {
2396 split_coords.x.dst0 += w;
2397 split_coords.x.dst1 = MIN2(split_coords.x.dst0 + w, orig->x.dst1);
2398 split_coords.y.dst0 = orig->y.dst0;
2399 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2400 adjust_split_source_coords(&orig->x, &split_coords.x, x_scale);
2401 } else {
2402 split_coords.y.dst0 += h;
2403 split_coords.y.dst1 = MIN2(split_coords.y.dst0 + h, orig->y.dst1);
2404 adjust_split_source_coords(&orig->y, &split_coords.y, y_scale);
2405 }
2406 } while (true);
2407 }
2408
2409 bool
blorp_blit_supports_compute(struct blorp_context * blorp,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_aux_usage dst_aux_usage)2410 blorp_blit_supports_compute(struct blorp_context *blorp,
2411 const struct isl_surf *src_surf,
2412 const struct isl_surf *dst_surf,
2413 enum isl_aux_usage dst_aux_usage)
2414 {
2415 /* Our compiler doesn't currently support typed image writes with MSAA.
2416 * Also, our BLORP compute shaders don't handle multisampling cases.
2417 */
2418 if (dst_surf->samples > 1 || src_surf->samples > 1)
2419 return false;
2420
2421 if (blorp->isl_dev->info->ver >= 12) {
2422 return dst_aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
2423 dst_aux_usage == ISL_AUX_USAGE_CCS_E ||
2424 dst_aux_usage == ISL_AUX_USAGE_NONE;
2425 } else if (blorp->isl_dev->info->ver >= 7) {
2426 return dst_aux_usage == ISL_AUX_USAGE_NONE;
2427 } else {
2428 /* No compute shader support */
2429 return false;
2430 }
2431 }
2432
2433 bool
blorp_blitter_supports_aux(const struct intel_device_info * devinfo,enum isl_aux_usage aux_usage)2434 blorp_blitter_supports_aux(const struct intel_device_info *devinfo,
2435 enum isl_aux_usage aux_usage)
2436 {
2437 switch (aux_usage) {
2438 case ISL_AUX_USAGE_NONE:
2439 return true;
2440 case ISL_AUX_USAGE_CCS_E:
2441 case ISL_AUX_USAGE_FCV_CCS_E:
2442 case ISL_AUX_USAGE_STC_CCS:
2443 return devinfo->verx10 >= 125;
2444 default:
2445 return false;
2446 }
2447 }
2448
2449 bool
blorp_copy_supports_blitter(struct blorp_context * blorp,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_aux_usage src_aux_usage,enum isl_aux_usage dst_aux_usage)2450 blorp_copy_supports_blitter(struct blorp_context *blorp,
2451 const struct isl_surf *src_surf,
2452 const struct isl_surf *dst_surf,
2453 enum isl_aux_usage src_aux_usage,
2454 enum isl_aux_usage dst_aux_usage)
2455 {
2456 const struct intel_device_info *devinfo = blorp->isl_dev->info;
2457
2458 if (devinfo->ver < 12)
2459 return false;
2460
2461 if (dst_surf->samples > 1 || src_surf->samples > 1)
2462 return false;
2463
2464 if (!blorp_blitter_supports_aux(devinfo, dst_aux_usage))
2465 return false;
2466
2467 if (!blorp_blitter_supports_aux(devinfo, src_aux_usage))
2468 return false;
2469
2470 const struct isl_format_layout *fmtl =
2471 isl_format_get_layout(dst_surf->format);
2472
2473 if (fmtl->bpb == 96) {
2474 /* XY_BLOCK_COPY_BLT mentions it doesn't support clear colors for 96bpp
2475 * formats, but none of them support CCS anyway, so it's a moot point.
2476 */
2477 assert(src_aux_usage == ISL_AUX_USAGE_NONE);
2478 assert(dst_aux_usage == ISL_AUX_USAGE_NONE);
2479
2480 /* We can only support linear mode for 96bpp. */
2481 if (src_surf->tiling != ISL_TILING_LINEAR ||
2482 dst_surf->tiling != ISL_TILING_LINEAR)
2483 return false;
2484 }
2485
2486 return true;
2487 }
2488
2489 void
blorp_blit(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,float src_layer,enum isl_format src_format,struct isl_swizzle src_swizzle,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,enum isl_format dst_format,struct isl_swizzle dst_swizzle,float src_x0,float src_y0,float src_x1,float src_y1,float dst_x0,float dst_y0,float dst_x1,float dst_y1,enum blorp_filter filter,bool mirror_x,bool mirror_y)2490 blorp_blit(struct blorp_batch *batch,
2491 const struct blorp_surf *src_surf,
2492 unsigned src_level, float src_layer,
2493 enum isl_format src_format, struct isl_swizzle src_swizzle,
2494 const struct blorp_surf *dst_surf,
2495 unsigned dst_level, unsigned dst_layer,
2496 enum isl_format dst_format, struct isl_swizzle dst_swizzle,
2497 float src_x0, float src_y0,
2498 float src_x1, float src_y1,
2499 float dst_x0, float dst_y0,
2500 float dst_x1, float dst_y1,
2501 enum blorp_filter filter,
2502 bool mirror_x, bool mirror_y)
2503 {
2504 struct blorp_params params;
2505 blorp_params_init(¶ms);
2506 params.op = BLORP_OP_BLIT;
2507 const bool compute = batch->flags & BLORP_BATCH_USE_COMPUTE;
2508 if (compute) {
2509 assert(blorp_blit_supports_compute(batch->blorp,
2510 src_surf->surf, dst_surf->surf,
2511 dst_surf->aux_usage));
2512 }
2513
2514 /* We cannot handle combined depth and stencil. */
2515 if (src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2516 assert(src_surf->surf->format == ISL_FORMAT_R8_UINT);
2517 if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT)
2518 assert(dst_surf->surf->format == ISL_FORMAT_R8_UINT);
2519
2520 if (dst_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT) {
2521 assert(src_surf->surf->usage & ISL_SURF_USAGE_STENCIL_BIT);
2522 /* Prior to Broadwell, we can't render to R8_UINT */
2523 if (batch->blorp->isl_dev->info->ver < 8) {
2524 src_format = ISL_FORMAT_R8_UNORM;
2525 dst_format = ISL_FORMAT_R8_UNORM;
2526 }
2527 }
2528
2529 blorp_surface_info_init(batch, ¶ms.src, src_surf, src_level,
2530 src_layer, src_format, false);
2531 blorp_surface_info_init(batch, ¶ms.dst, dst_surf, dst_level,
2532 dst_layer, dst_format, true);
2533
2534 params.src.view.swizzle = src_swizzle;
2535 params.dst.view.swizzle = dst_swizzle;
2536
2537 const struct isl_format_layout *src_fmtl =
2538 isl_format_get_layout(params.src.view.format);
2539
2540 struct blorp_blit_prog_key key = {
2541 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_BLIT),
2542 .base.shader_pipeline = compute ? BLORP_SHADER_PIPELINE_COMPUTE :
2543 BLORP_SHADER_PIPELINE_RENDER,
2544 .filter = filter,
2545 .sint32_to_uint = src_fmtl->channels.r.bits == 32 &&
2546 isl_format_has_sint_channel(params.src.view.format) &&
2547 isl_format_has_uint_channel(params.dst.view.format),
2548 .uint32_to_sint = src_fmtl->channels.r.bits == 32 &&
2549 isl_format_has_uint_channel(params.src.view.format) &&
2550 isl_format_has_sint_channel(params.dst.view.format),
2551 };
2552
2553 params.shader_type = key.base.shader_type;
2554 params.shader_pipeline = key.base.shader_pipeline;
2555
2556 /* Scaling factors used for bilinear filtering in multisample scaled
2557 * blits.
2558 */
2559 if (params.src.surf.samples == 16)
2560 key.x_scale = 4.0f;
2561 else
2562 key.x_scale = 2.0f;
2563 key.y_scale = params.src.surf.samples / key.x_scale;
2564
2565 params.wm_inputs.rect_grid.x1 =
2566 u_minify(params.src.surf.logical_level0_px.width, src_level) *
2567 key.x_scale - 1.0f;
2568 params.wm_inputs.rect_grid.y1 =
2569 u_minify(params.src.surf.logical_level0_px.height, src_level) *
2570 key.y_scale - 1.0f;
2571
2572 struct blt_coords coords = {
2573 .x = {
2574 .src0 = src_x0,
2575 .src1 = src_x1,
2576 .dst0 = dst_x0,
2577 .dst1 = dst_x1,
2578 .mirror = mirror_x
2579 },
2580 .y = {
2581 .src0 = src_y0,
2582 .src1 = src_y1,
2583 .dst0 = dst_y0,
2584 .dst1 = dst_y1,
2585 .mirror = mirror_y
2586 }
2587 };
2588
2589 do_blorp_blit(batch, ¶ms, &key, &coords);
2590 }
2591
2592 static enum isl_format
get_copy_format_for_bpb(const struct isl_device * isl_dev,unsigned bpb)2593 get_copy_format_for_bpb(const struct isl_device *isl_dev, unsigned bpb)
2594 {
2595 /* The choice of UNORM and UINT formats is very intentional here. Most
2596 * of the time, we want to use a UINT format to avoid any rounding error
2597 * in the blit. For stencil blits, R8_UINT is required by the hardware.
2598 * (It's the only format allowed in conjunction with W-tiling.) Also we
2599 * intentionally use the 4-channel formats whenever we can. This is so
2600 * that, when we do a RGB <-> RGBX copy, the two formats will line up
2601 * even though one of them is 3/4 the size of the other. The choice of
2602 * UNORM vs. UINT is also very intentional because we don't have 8 or
2603 * 16-bit RGB UINT formats until Sky Lake so we have to use UNORM there.
2604 * Fortunately, the only time we should ever use two different formats in
2605 * the table below is for RGB -> RGBA blits and so we will never have any
2606 * UNORM/UINT mismatch.
2607 */
2608 if (ISL_GFX_VER(isl_dev) >= 9) {
2609 switch (bpb) {
2610 case 8: return ISL_FORMAT_R8_UINT;
2611 case 16: return ISL_FORMAT_R8G8_UINT;
2612 case 24: return ISL_FORMAT_R8G8B8_UINT;
2613 case 32: return ISL_FORMAT_R8G8B8A8_UINT;
2614 case 48: return ISL_FORMAT_R16G16B16_UINT;
2615 case 64: return ISL_FORMAT_R16G16B16A16_UINT;
2616 case 96: return ISL_FORMAT_R32G32B32_UINT;
2617 case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2618 default:
2619 unreachable("Unknown format bpb");
2620 }
2621 } else {
2622 switch (bpb) {
2623 case 8: return ISL_FORMAT_R8_UINT;
2624 case 16: return ISL_FORMAT_R8G8_UINT;
2625 case 24: return ISL_FORMAT_R8G8B8_UNORM;
2626 case 32: return ISL_FORMAT_R8G8B8A8_UNORM;
2627 case 48: return ISL_FORMAT_R16G16B16_UNORM;
2628 case 64: return ISL_FORMAT_R16G16B16A16_UNORM;
2629 case 96: return ISL_FORMAT_R32G32B32_UINT;
2630 case 128:return ISL_FORMAT_R32G32B32A32_UINT;
2631 default:
2632 unreachable("Unknown format bpb");
2633 }
2634 }
2635 }
2636
2637 /** Returns a UINT format that is CCS-compatible with the given format
2638 *
2639 * The PRM's say absolutely nothing about how render compression works. The
2640 * only thing they provide is a list of formats on which it is and is not
2641 * supported. Empirical testing indicates that the compression is only based
2642 * on the bit-layout of the format and the channel encoding doesn't matter.
2643 * So, while texture views don't work in general, you can create a view as
2644 * long as the bit-layout of the formats are the same.
2645 *
2646 * Fortunately, for every render compression capable format, the UINT format
2647 * with the same bit layout also supports render compression. This means that
2648 * we only need to handle UINT formats for copy operations. In order to do
2649 * copies between formats with different bit layouts, we attach both with a
2650 * UINT format and use bit_cast_color() to generate code to do the bit-cast
2651 * operation between the two bit layouts.
2652 */
2653 static enum isl_format
get_ccs_compatible_copy_format(const struct isl_format_layout * fmtl)2654 get_ccs_compatible_copy_format(const struct isl_format_layout *fmtl)
2655 {
2656 switch (fmtl->format) {
2657 case ISL_FORMAT_R32G32B32A32_FLOAT:
2658 case ISL_FORMAT_R32G32B32A32_SINT:
2659 case ISL_FORMAT_R32G32B32A32_UINT:
2660 case ISL_FORMAT_R32G32B32A32_UNORM:
2661 case ISL_FORMAT_R32G32B32A32_SNORM:
2662 case ISL_FORMAT_R32G32B32X32_FLOAT:
2663 return ISL_FORMAT_R32G32B32A32_UINT;
2664
2665 case ISL_FORMAT_R16G16B16A16_UNORM:
2666 case ISL_FORMAT_R16G16B16A16_SNORM:
2667 case ISL_FORMAT_R16G16B16A16_SINT:
2668 case ISL_FORMAT_R16G16B16A16_UINT:
2669 case ISL_FORMAT_R16G16B16A16_FLOAT:
2670 case ISL_FORMAT_R16G16B16X16_UNORM:
2671 case ISL_FORMAT_R16G16B16X16_FLOAT:
2672 return ISL_FORMAT_R16G16B16A16_UINT;
2673
2674 case ISL_FORMAT_R32G32_FLOAT:
2675 case ISL_FORMAT_R32G32_SINT:
2676 case ISL_FORMAT_R32G32_UINT:
2677 case ISL_FORMAT_R32G32_UNORM:
2678 case ISL_FORMAT_R32G32_SNORM:
2679 return ISL_FORMAT_R32G32_UINT;
2680
2681 case ISL_FORMAT_B8G8R8A8_UNORM:
2682 case ISL_FORMAT_B8G8R8A8_UNORM_SRGB:
2683 case ISL_FORMAT_R8G8B8A8_UNORM:
2684 case ISL_FORMAT_R8G8B8A8_UNORM_SRGB:
2685 case ISL_FORMAT_R8G8B8A8_SNORM:
2686 case ISL_FORMAT_R8G8B8A8_SINT:
2687 case ISL_FORMAT_R8G8B8A8_UINT:
2688 case ISL_FORMAT_B8G8R8X8_UNORM:
2689 case ISL_FORMAT_B8G8R8X8_UNORM_SRGB:
2690 case ISL_FORMAT_R8G8B8X8_UNORM:
2691 case ISL_FORMAT_R8G8B8X8_UNORM_SRGB:
2692 return ISL_FORMAT_R8G8B8A8_UINT;
2693
2694 case ISL_FORMAT_R16G16_UNORM:
2695 case ISL_FORMAT_R16G16_SNORM:
2696 case ISL_FORMAT_R16G16_SINT:
2697 case ISL_FORMAT_R16G16_UINT:
2698 case ISL_FORMAT_R16G16_FLOAT:
2699 return ISL_FORMAT_R16G16_UINT;
2700
2701 case ISL_FORMAT_R32_SINT:
2702 case ISL_FORMAT_R32_UINT:
2703 case ISL_FORMAT_R32_FLOAT:
2704 case ISL_FORMAT_R32_UNORM:
2705 case ISL_FORMAT_R32_SNORM:
2706 return ISL_FORMAT_R32_UINT;
2707
2708 case ISL_FORMAT_R11G11B10_FLOAT:
2709 return ISL_FORMAT_R8G8B8A8_UINT;
2710
2711 case ISL_FORMAT_B10G10R10A2_UNORM:
2712 case ISL_FORMAT_B10G10R10A2_UNORM_SRGB:
2713 case ISL_FORMAT_R10G10B10A2_UNORM:
2714 case ISL_FORMAT_R10G10B10A2_UNORM_SRGB:
2715 case ISL_FORMAT_R10G10B10_FLOAT_A2_UNORM:
2716 case ISL_FORMAT_R10G10B10A2_UINT:
2717 return ISL_FORMAT_R10G10B10A2_UINT;
2718
2719 case ISL_FORMAT_R16_UNORM:
2720 case ISL_FORMAT_R16_SNORM:
2721 case ISL_FORMAT_R16_SINT:
2722 case ISL_FORMAT_R16_UINT:
2723 case ISL_FORMAT_R16_FLOAT:
2724 return ISL_FORMAT_R16_UINT;
2725
2726 case ISL_FORMAT_R8G8_UNORM:
2727 case ISL_FORMAT_R8G8_SNORM:
2728 case ISL_FORMAT_R8G8_SINT:
2729 case ISL_FORMAT_R8G8_UINT:
2730 return ISL_FORMAT_R8G8_UINT;
2731
2732 case ISL_FORMAT_YCRCB_NORMAL:
2733 case ISL_FORMAT_YCRCB_SWAPY:
2734 case ISL_FORMAT_YCRCB_SWAPUV:
2735 case ISL_FORMAT_YCRCB_SWAPUVY:
2736 /* Tiger Lake starts claiming CCS_E support for certain YCRCB formats.
2737 * BLORP chooses to take the CCS-compatible format path whenever ISL
2738 * claims CCS_E support on a format, not when CCS_E is actually used.
2739 * Therefore, if these formats are going to be used with BLORP, we need
2740 * a CCS-compatible format. R8G8_UINT seems as good as any.
2741 */
2742 return ISL_FORMAT_R8G8_UINT;
2743
2744 case ISL_FORMAT_B5G5R5X1_UNORM:
2745 case ISL_FORMAT_B5G5R5X1_UNORM_SRGB:
2746 case ISL_FORMAT_B5G5R5A1_UNORM:
2747 case ISL_FORMAT_B5G5R5A1_UNORM_SRGB:
2748 return ISL_FORMAT_B5G5R5A1_UNORM;
2749
2750 case ISL_FORMAT_A4B4G4R4_UNORM:
2751 case ISL_FORMAT_B4G4R4A4_UNORM:
2752 case ISL_FORMAT_B4G4R4A4_UNORM_SRGB:
2753 return ISL_FORMAT_B4G4R4A4_UNORM;
2754
2755 case ISL_FORMAT_B5G6R5_UNORM:
2756 case ISL_FORMAT_B5G6R5_UNORM_SRGB:
2757 return ISL_FORMAT_B5G6R5_UNORM;
2758
2759 case ISL_FORMAT_A1B5G5R5_UNORM:
2760 return ISL_FORMAT_A1B5G5R5_UNORM;
2761
2762 case ISL_FORMAT_A8_UNORM:
2763 case ISL_FORMAT_R8_UNORM:
2764 case ISL_FORMAT_R8_SNORM:
2765 case ISL_FORMAT_R8_SINT:
2766 case ISL_FORMAT_R8_UINT:
2767 return ISL_FORMAT_R8_UINT;
2768
2769 default:
2770 unreachable("Not a compressible format");
2771 }
2772 }
2773
2774 void
blorp_surf_convert_to_uncompressed(const struct isl_device * isl_dev,struct blorp_surface_info * info,uint32_t * x,uint32_t * y,uint32_t * width,uint32_t * height)2775 blorp_surf_convert_to_uncompressed(const struct isl_device *isl_dev,
2776 struct blorp_surface_info *info,
2777 uint32_t *x, uint32_t *y,
2778 uint32_t *width, uint32_t *height)
2779 {
2780 const struct isl_format_layout *fmtl =
2781 isl_format_get_layout(info->surf.format);
2782
2783 assert(fmtl->bw > 1 || fmtl->bh > 1);
2784
2785 /* This should be the first modification made to the surface */
2786 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
2787
2788 if (width && height) {
2789 ASSERTED const uint32_t level_width =
2790 u_minify(info->surf.logical_level0_px.width, info->view.base_level);
2791 ASSERTED const uint32_t level_height =
2792 u_minify(info->surf.logical_level0_px.height, info->view.base_level);
2793 assert(*width % fmtl->bw == 0 || *x + *width == level_width);
2794 assert(*height % fmtl->bh == 0 || *y + *height == level_height);
2795 *width = DIV_ROUND_UP(*width, fmtl->bw);
2796 *height = DIV_ROUND_UP(*height, fmtl->bh);
2797 }
2798
2799 if (x && y) {
2800 assert(*x % fmtl->bw == 0);
2801 assert(*y % fmtl->bh == 0);
2802 *x /= fmtl->bw;
2803 *y /= fmtl->bh;
2804 }
2805
2806 /* We only want one level and slice */
2807 info->view.levels = 1;
2808 info->view.array_len = 1;
2809
2810 if (info->surf.dim == ISL_SURF_DIM_3D) {
2811 /* Roll the Z offset into the image view */
2812 info->view.base_array_layer += info->z_offset;
2813 info->z_offset = 0;
2814 }
2815
2816 uint64_t offset_B;
2817 ASSERTED bool ok =
2818 isl_surf_get_uncompressed_surf(isl_dev, &info->surf, &info->view,
2819 &info->surf, &info->view, &offset_B,
2820 &info->tile_x_sa, &info->tile_y_sa);
2821 assert(ok);
2822 info->addr.offset += offset_B;
2823
2824 /* BLORP doesn't use the actual intratile offsets. Instead, it needs the
2825 * surface to be a bit bigger and we offset the vertices instead. Standard
2826 * tilings don't need intratile offsets because each subresource is aligned
2827 * to a bpb-based tile boundary or miptail slot offset.
2828 */
2829 if (isl_tiling_is_64(info->surf.tiling) ||
2830 isl_tiling_is_std_y(info->surf.tiling)) {
2831 assert(info->tile_x_sa == 0 && info->tile_y_sa == 0);
2832 } else {
2833 assert(info->surf.dim == ISL_SURF_DIM_2D);
2834 assert(info->surf.logical_level0_px.array_len == 1);
2835 info->surf.logical_level0_px.w += info->tile_x_sa;
2836 info->surf.logical_level0_px.h += info->tile_y_sa;
2837 info->surf.phys_level0_sa.w += info->tile_x_sa;
2838 info->surf.phys_level0_sa.h += info->tile_y_sa;
2839 }
2840 }
2841
2842 bool
blorp_copy_supports_compute(struct blorp_context * blorp,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_aux_usage dst_aux_usage)2843 blorp_copy_supports_compute(struct blorp_context *blorp,
2844 const struct isl_surf *src_surf,
2845 const struct isl_surf *dst_surf,
2846 enum isl_aux_usage dst_aux_usage)
2847 {
2848 return blorp_blit_supports_compute(blorp, src_surf, dst_surf, dst_aux_usage);
2849 }
2850
2851 void
blorp_copy_get_formats(const struct isl_device * isl_dev,const struct isl_surf * src_surf,const struct isl_surf * dst_surf,enum isl_format * src_view_format,enum isl_format * dst_view_format)2852 blorp_copy_get_formats(const struct isl_device *isl_dev,
2853 const struct isl_surf *src_surf,
2854 const struct isl_surf *dst_surf,
2855 enum isl_format *src_view_format,
2856 enum isl_format *dst_view_format)
2857 {
2858 const struct isl_format_layout *src_fmtl =
2859 isl_format_get_layout(src_surf->format);
2860 const struct isl_format_layout *dst_fmtl =
2861 isl_format_get_layout(dst_surf->format);
2862
2863 if (ISL_GFX_VER(isl_dev) >= 8 &&
2864 isl_surf_usage_is_depth(src_surf->usage)) {
2865 /* In order to use HiZ, we have to use the real format for the source.
2866 * Depth <-> Color copies are not allowed.
2867 */
2868 *src_view_format = src_surf->format;
2869 *dst_view_format = src_surf->format;
2870 } else if (ISL_GFX_VER(isl_dev) >= 7 &&
2871 isl_surf_usage_is_depth(dst_surf->usage)) {
2872 /* On Gfx7 and higher, we use actual depth writes for blits into depth
2873 * buffers so we need the real format.
2874 */
2875 *src_view_format = dst_surf->format;
2876 *dst_view_format = dst_surf->format;
2877 } else if (isl_surf_usage_is_depth(src_surf->usage) ||
2878 isl_surf_usage_is_depth(dst_surf->usage)) {
2879 assert(src_fmtl->bpb == dst_fmtl->bpb);
2880 *src_view_format =
2881 *dst_view_format =
2882 get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2883 } else if (ISL_GFX_VER(isl_dev) < 20 &&
2884 isl_format_supports_ccs_e(isl_dev->info, dst_surf->format)) {
2885 *dst_view_format = get_ccs_compatible_copy_format(dst_fmtl);
2886 if (isl_format_supports_ccs_e(isl_dev->info, src_surf->format)) {
2887 *src_view_format = get_ccs_compatible_copy_format(src_fmtl);
2888 } else if (src_fmtl->bpb == dst_fmtl->bpb) {
2889 *src_view_format = *dst_view_format;
2890 } else {
2891 *src_view_format = get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
2892 }
2893 } else if (ISL_GFX_VER(isl_dev) < 20 &&
2894 isl_format_supports_ccs_e(isl_dev->info, src_surf->format)) {
2895 *src_view_format = get_ccs_compatible_copy_format(src_fmtl);
2896 if (src_fmtl->bpb == dst_fmtl->bpb) {
2897 *dst_view_format = *src_view_format;
2898 } else {
2899 *dst_view_format = get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2900 }
2901 } else {
2902 *dst_view_format = get_copy_format_for_bpb(isl_dev, dst_fmtl->bpb);
2903 *src_view_format = get_copy_format_for_bpb(isl_dev, src_fmtl->bpb);
2904 }
2905 }
2906
2907
2908 void
blorp_copy(struct blorp_batch * batch,const struct blorp_surf * src_surf,unsigned src_level,unsigned src_layer,const struct blorp_surf * dst_surf,unsigned dst_level,unsigned dst_layer,uint32_t src_x,uint32_t src_y,uint32_t dst_x,uint32_t dst_y,uint32_t src_width,uint32_t src_height)2909 blorp_copy(struct blorp_batch *batch,
2910 const struct blorp_surf *src_surf,
2911 unsigned src_level, unsigned src_layer,
2912 const struct blorp_surf *dst_surf,
2913 unsigned dst_level, unsigned dst_layer,
2914 uint32_t src_x, uint32_t src_y,
2915 uint32_t dst_x, uint32_t dst_y,
2916 uint32_t src_width, uint32_t src_height)
2917 {
2918 const struct isl_device *isl_dev = batch->blorp->isl_dev;
2919 const struct intel_device_info *devinfo = isl_dev->info;
2920 struct blorp_params params;
2921
2922 if (src_width == 0 || src_height == 0)
2923 return;
2924
2925 blorp_params_init(¶ms);
2926 params.op = BLORP_OP_COPY;
2927
2928 const bool compute = batch->flags & BLORP_BATCH_USE_COMPUTE;
2929 if (compute) {
2930 assert(blorp_copy_supports_compute(batch->blorp,
2931 src_surf->surf, dst_surf->surf,
2932 dst_surf->aux_usage));
2933 } else if (batch->flags & BLORP_BATCH_USE_BLITTER) {
2934 assert(blorp_copy_supports_blitter(batch->blorp,
2935 src_surf->surf, dst_surf->surf,
2936 src_surf->aux_usage,
2937 dst_surf->aux_usage));
2938 }
2939
2940 blorp_surface_info_init(batch, ¶ms.src, src_surf, src_level,
2941 src_layer, ISL_FORMAT_UNSUPPORTED, false);
2942 blorp_surface_info_init(batch, ¶ms.dst, dst_surf, dst_level,
2943 dst_layer, ISL_FORMAT_UNSUPPORTED, true);
2944
2945 struct blorp_blit_prog_key key = {
2946 .base = BLORP_BASE_KEY_INIT(BLORP_SHADER_TYPE_COPY),
2947 .base.shader_pipeline = compute ? BLORP_SHADER_PIPELINE_COMPUTE :
2948 BLORP_SHADER_PIPELINE_RENDER,
2949 .filter = BLORP_FILTER_NONE,
2950 .need_src_offset = src_surf->tile_x_sa || src_surf->tile_y_sa,
2951 .need_dst_offset = dst_surf->tile_x_sa || dst_surf->tile_y_sa,
2952 };
2953
2954 params.shader_type = key.base.shader_type;
2955 params.shader_pipeline = key.base.shader_pipeline;
2956
2957 const struct isl_format_layout *src_fmtl =
2958 isl_format_get_layout(params.src.surf.format);
2959 const struct isl_format_layout *dst_fmtl =
2960 isl_format_get_layout(params.dst.surf.format);
2961
2962 assert(params.src.aux_usage == ISL_AUX_USAGE_NONE ||
2963 params.src.aux_usage == ISL_AUX_USAGE_HIZ ||
2964 params.src.aux_usage == ISL_AUX_USAGE_HIZ_CCS_WT ||
2965 params.src.aux_usage == ISL_AUX_USAGE_MCS ||
2966 params.src.aux_usage == ISL_AUX_USAGE_MCS_CCS ||
2967 params.src.aux_usage == ISL_AUX_USAGE_CCS_E ||
2968 params.src.aux_usage == ISL_AUX_USAGE_FCV_CCS_E ||
2969 params.src.aux_usage == ISL_AUX_USAGE_STC_CCS);
2970
2971 blorp_copy_get_formats(isl_dev, ¶ms.src.surf, ¶ms.dst.surf,
2972 ¶ms.src.view.format, ¶ms.dst.view.format);
2973
2974 if (isl_aux_usage_has_fast_clears(params.src.aux_usage) &&
2975 isl_dev->ss.clear_color_state_size > 0) {
2976 /* Depending on the format, the sampler may change the location from
2977 * which it fetches the clear color. This can be a problem in some
2978 * cases, so make sure that the view format won't change the location.
2979 */
2980 ASSERTED enum isl_format src_view_fmt = params.src.view.format;
2981 ASSERTED enum isl_format src_surf_fmt = params.src.surf.format;
2982 assert(isl_get_sampler_clear_field_offset(devinfo, src_view_fmt) ==
2983 isl_get_sampler_clear_field_offset(devinfo, src_surf_fmt));
2984 }
2985
2986 if (params.src.view.format != params.dst.view.format) {
2987 enum isl_format src_cast_format = params.src.view.format;
2988 enum isl_format dst_cast_format = params.dst.view.format;
2989
2990 /* The BLORP bitcast code gets confused by RGB formats. Just treat them
2991 * as RGBA and then everything will be happy. This is perfectly safe
2992 * because BLORP likes to treat things as if they have vec4 colors all
2993 * the time anyway.
2994 */
2995 if (isl_format_get_layout(src_cast_format)->bpb % 3 == 0)
2996 src_cast_format = isl_format_rgb_to_rgba(src_cast_format);
2997 if (isl_format_get_layout(dst_cast_format)->bpb % 3 == 0)
2998 dst_cast_format = isl_format_rgb_to_rgba(dst_cast_format);
2999
3000 if (src_cast_format != dst_cast_format) {
3001 key.format_bit_cast = true;
3002 key.src_format = src_cast_format;
3003 key.dst_format = dst_cast_format;
3004 }
3005 }
3006
3007 if (src_fmtl->bw > 1 || src_fmtl->bh > 1) {
3008 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.src,
3009 &src_x, &src_y,
3010 &src_width, &src_height);
3011 key.need_src_offset = true;
3012 }
3013
3014 if (dst_fmtl->bw > 1 || dst_fmtl->bh > 1) {
3015 blorp_surf_convert_to_uncompressed(batch->blorp->isl_dev, ¶ms.dst,
3016 &dst_x, &dst_y, NULL, NULL);
3017 key.need_dst_offset = true;
3018 }
3019
3020 /* Once both surfaces are stompped to uncompressed as needed, the
3021 * destination size is the same as the source size.
3022 */
3023 uint32_t dst_width = src_width;
3024 uint32_t dst_height = src_height;
3025
3026 if (batch->flags & BLORP_BATCH_USE_BLITTER) {
3027 if (devinfo->verx10 < 125) {
3028 blorp_surf_convert_to_single_slice(isl_dev, ¶ms.dst);
3029 blorp_surf_convert_to_single_slice(isl_dev, ¶ms.src);
3030 }
3031
3032 params.x0 = dst_x;
3033 params.x1 = dst_x + dst_width;
3034 params.y0 = dst_y;
3035 params.y1 = dst_y + dst_height;
3036 params.wm_inputs.coord_transform[0].offset = dst_x - (float)src_x;
3037 params.wm_inputs.coord_transform[1].offset = dst_y - (float)src_y;
3038 params.wm_inputs.coord_transform[0].multiplier = 1.0f;
3039 params.wm_inputs.coord_transform[1].multiplier = 1.0f;
3040
3041 batch->blorp->exec(batch, ¶ms);
3042 return;
3043 }
3044
3045 struct blt_coords coords = {
3046 .x = {
3047 .src0 = src_x,
3048 .src1 = src_x + src_width,
3049 .dst0 = dst_x,
3050 .dst1 = dst_x + dst_width,
3051 .mirror = false
3052 },
3053 .y = {
3054 .src0 = src_y,
3055 .src1 = src_y + src_height,
3056 .dst0 = dst_y,
3057 .dst1 = dst_y + dst_height,
3058 .mirror = false
3059 }
3060 };
3061
3062 do_blorp_blit(batch, ¶ms, &key, &coords);
3063 }
3064
3065 static enum isl_format
isl_format_for_size(unsigned size_B)3066 isl_format_for_size(unsigned size_B)
3067 {
3068 switch (size_B) {
3069 case 1: return ISL_FORMAT_R8_UINT;
3070 case 2: return ISL_FORMAT_R8G8_UINT;
3071 case 4: return ISL_FORMAT_R8G8B8A8_UINT;
3072 case 8: return ISL_FORMAT_R16G16B16A16_UINT;
3073 case 16: return ISL_FORMAT_R32G32B32A32_UINT;
3074 default:
3075 unreachable("Not a power-of-two format size");
3076 }
3077 }
3078
3079 /**
3080 * Returns the greatest common divisor of a and b that is a power of two.
3081 */
3082 static uint64_t
gcd_pow2_u64(uint64_t a,uint64_t b)3083 gcd_pow2_u64(uint64_t a, uint64_t b)
3084 {
3085 assert(a > 0 || b > 0);
3086
3087 unsigned a_log2 = ffsll(a) - 1;
3088 unsigned b_log2 = ffsll(b) - 1;
3089
3090 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which
3091 * case, the MIN2() will take the other one. If both are 0 then we will
3092 * hit the assert above.
3093 */
3094 return 1 << MIN2(a_log2, b_log2);
3095 }
3096
3097 static void
do_buffer_copy(struct blorp_batch * batch,struct blorp_address * src,struct blorp_address * dst,int width,int height,int block_size)3098 do_buffer_copy(struct blorp_batch *batch,
3099 struct blorp_address *src,
3100 struct blorp_address *dst,
3101 int width, int height, int block_size)
3102 {
3103 /* The actual format we pick doesn't matter as blorp will throw it away.
3104 * The only thing that actually matters is the size.
3105 */
3106 enum isl_format format = isl_format_for_size(block_size);
3107
3108 UNUSED bool ok;
3109 struct isl_surf surf;
3110 ok = isl_surf_init(batch->blorp->isl_dev, &surf,
3111 .dim = ISL_SURF_DIM_2D,
3112 .format = format,
3113 .width = width,
3114 .height = height,
3115 .depth = 1,
3116 .levels = 1,
3117 .array_len = 1,
3118 .samples = 1,
3119 .row_pitch_B = width * block_size,
3120 .usage = ISL_SURF_USAGE_TEXTURE_BIT |
3121 ISL_SURF_USAGE_RENDER_TARGET_BIT,
3122 .tiling_flags = ISL_TILING_LINEAR_BIT);
3123 assert(ok);
3124
3125 struct blorp_surf src_blorp_surf = {
3126 .surf = &surf,
3127 .addr = *src,
3128 };
3129
3130 struct blorp_surf dst_blorp_surf = {
3131 .surf = &surf,
3132 .addr = *dst,
3133 };
3134
3135 blorp_copy(batch, &src_blorp_surf, 0, 0, &dst_blorp_surf, 0, 0,
3136 0, 0, 0, 0, width, height);
3137 }
3138
3139 void
blorp_buffer_copy(struct blorp_batch * batch,struct blorp_address src,struct blorp_address dst,uint64_t size)3140 blorp_buffer_copy(struct blorp_batch *batch,
3141 struct blorp_address src,
3142 struct blorp_address dst,
3143 uint64_t size)
3144 {
3145 const struct intel_device_info *devinfo = batch->blorp->isl_dev->info;
3146 uint64_t copy_size = size;
3147
3148 /* This is maximum possible width/height our HW can handle */
3149 uint64_t max_surface_dim = 1 << (devinfo->ver >= 7 ? 14 : 13);
3150
3151 /* First, we compute the biggest format that can be used with the
3152 * given offsets and size.
3153 */
3154 int bs = 16;
3155 bs = gcd_pow2_u64(bs, src.offset);
3156 bs = gcd_pow2_u64(bs, dst.offset);
3157 bs = gcd_pow2_u64(bs, size);
3158
3159 /* First, we make a bunch of max-sized copies */
3160 uint64_t max_copy_size = max_surface_dim * max_surface_dim * bs;
3161 while (copy_size >= max_copy_size) {
3162 do_buffer_copy(batch, &src, &dst, max_surface_dim, max_surface_dim, bs);
3163 copy_size -= max_copy_size;
3164 src.offset += max_copy_size;
3165 dst.offset += max_copy_size;
3166 }
3167
3168 /* Now make a max-width copy */
3169 uint64_t height = copy_size / (max_surface_dim * bs);
3170 assert(height < max_surface_dim);
3171 if (height != 0) {
3172 uint64_t rect_copy_size = height * max_surface_dim * bs;
3173 do_buffer_copy(batch, &src, &dst, max_surface_dim, height, bs);
3174 copy_size -= rect_copy_size;
3175 src.offset += rect_copy_size;
3176 dst.offset += rect_copy_size;
3177 }
3178
3179 /* Finally, make a small copy to finish it off */
3180 if (copy_size != 0) {
3181 do_buffer_copy(batch, &src, &dst, copy_size / bs, 1, bs);
3182 }
3183 }
3184