1 /*
2 * Copyright (C) 2008 VMware, Inc.
3 * Copyright (C) 2014 Broadcom
4 * Copyright (C) 2018-2019 Alyssa Rosenzweig
5 * Copyright (C) 2019-2020 Collabora, Ltd.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 * SOFTWARE.
25 *
26 */
27
28 #include "pan_texture.h"
29 #include "util/macros.h"
30 #include "util/u_math.h"
31
32 #if PAN_ARCH >= 5
33 /*
34 * Arm Scalable Texture Compression (ASTC) corresponds to just a few formats.
35 * The block dimension is not part of the format. Instead, it is encoded as a
36 * 6-bit tag on the payload pointer. Map the block size for a single dimension.
37 */
38 static inline enum mali_astc_2d_dimension
panfrost_astc_dim_2d(unsigned dim)39 panfrost_astc_dim_2d(unsigned dim)
40 {
41 switch (dim) {
42 case 4:
43 return MALI_ASTC_2D_DIMENSION_4;
44 case 5:
45 return MALI_ASTC_2D_DIMENSION_5;
46 case 6:
47 return MALI_ASTC_2D_DIMENSION_6;
48 case 8:
49 return MALI_ASTC_2D_DIMENSION_8;
50 case 10:
51 return MALI_ASTC_2D_DIMENSION_10;
52 case 12:
53 return MALI_ASTC_2D_DIMENSION_12;
54 default:
55 unreachable("Invalid ASTC dimension");
56 }
57 }
58
59 static inline enum mali_astc_3d_dimension
panfrost_astc_dim_3d(unsigned dim)60 panfrost_astc_dim_3d(unsigned dim)
61 {
62 switch (dim) {
63 case 3:
64 return MALI_ASTC_3D_DIMENSION_3;
65 case 4:
66 return MALI_ASTC_3D_DIMENSION_4;
67 case 5:
68 return MALI_ASTC_3D_DIMENSION_5;
69 case 6:
70 return MALI_ASTC_3D_DIMENSION_6;
71 default:
72 unreachable("Invalid ASTC dimension");
73 }
74 }
75 #endif
76
77 /* Texture addresses are tagged with information about compressed formats.
78 * AFBC uses a bit for whether the colorspace transform is enabled (RGB and
79 * RGBA only).
80 * For ASTC, this is a "stretch factor" encoding the block size. */
81
82 static unsigned
panfrost_compression_tag(const struct util_format_description * desc,enum mali_texture_dimension dim,uint64_t modifier)83 panfrost_compression_tag(const struct util_format_description *desc,
84 enum mali_texture_dimension dim, uint64_t modifier)
85 {
86 #if PAN_ARCH >= 5 && PAN_ARCH <= 8
87 if (drm_is_afbc(modifier)) {
88 unsigned flags =
89 (modifier & AFBC_FORMAT_MOD_YTR) ? MALI_AFBC_SURFACE_FLAG_YTR : 0;
90
91 #if PAN_ARCH >= 6
92 /* Prefetch enable */
93 flags |= MALI_AFBC_SURFACE_FLAG_PREFETCH;
94
95 if (panfrost_afbc_is_wide(modifier))
96 flags |= MALI_AFBC_SURFACE_FLAG_WIDE_BLOCK;
97 #endif
98
99 #if PAN_ARCH >= 7
100 /* Tiled headers */
101 if (modifier & AFBC_FORMAT_MOD_TILED)
102 flags |= MALI_AFBC_SURFACE_FLAG_TILED_HEADER;
103
104 /* Used to make sure AFBC headers don't point outside the AFBC
105 * body. HW is using the AFBC surface stride to do this check,
106 * which doesn't work for 3D textures because the surface
107 * stride does not cover the body. Only supported on v7+.
108 */
109 if (dim != MALI_TEXTURE_DIMENSION_3D)
110 flags |= MALI_AFBC_SURFACE_FLAG_CHECK_PAYLOAD_RANGE;
111 #endif
112
113 return flags;
114 } else if (desc->layout == UTIL_FORMAT_LAYOUT_ASTC) {
115 if (desc->block.depth > 1) {
116 return (panfrost_astc_dim_3d(desc->block.depth) << 4) |
117 (panfrost_astc_dim_3d(desc->block.height) << 2) |
118 panfrost_astc_dim_3d(desc->block.width);
119 } else {
120 return (panfrost_astc_dim_2d(desc->block.height) << 3) |
121 panfrost_astc_dim_2d(desc->block.width);
122 }
123 }
124 #endif
125
126 /* Tags are not otherwise used */
127 return 0;
128 }
129
130 /* Following the texture descriptor is a number of descriptors. How many? */
131
132 static unsigned
panfrost_texture_num_elements(const struct pan_image_view * iview)133 panfrost_texture_num_elements(const struct pan_image_view *iview)
134 {
135 unsigned levels = 1 + iview->last_level - iview->first_level;
136 unsigned layers = 1 + iview->last_layer - iview->first_layer;
137 unsigned nr_samples = pan_image_view_get_nr_samples(iview);
138
139 return levels * layers * MAX2(nr_samples, 1);
140 }
141
142 /* Conservative estimate of the size of the texture payload a priori.
143 * Average case, size equal to the actual size. Worst case, off by 2x (if
144 * a manual stride is not needed on a linear texture). Returned value
145 * must be greater than or equal to the actual size, so it's safe to use
146 * as an allocation amount */
147
148 unsigned
GENX(panfrost_estimate_texture_payload_size)149 GENX(panfrost_estimate_texture_payload_size)(const struct pan_image_view *iview)
150 {
151 size_t element_size;
152
153 #if PAN_ARCH >= 9
154 element_size = pan_size(PLANE);
155
156 /* 2-plane and 3-plane YUV use two plane descriptors. */
157 if (panfrost_format_is_yuv(iview->format) && iview->planes[1] != NULL)
158 element_size *= 2;
159 #elif PAN_ARCH == 7
160 if (panfrost_format_is_yuv(iview->format))
161 element_size = pan_size(MULTIPLANAR_SURFACE);
162 else
163 element_size = pan_size(SURFACE_WITH_STRIDE);
164 #else
165 /* Assume worst case. Overestimates on Midgard, but that's ok. */
166 element_size = pan_size(SURFACE_WITH_STRIDE);
167 #endif
168
169 unsigned elements = panfrost_texture_num_elements(iview);
170
171 return element_size * elements;
172 }
173
174 static void
panfrost_get_surface_strides(const struct pan_image_layout * layout,unsigned l,int32_t * row_stride,int32_t * surf_stride)175 panfrost_get_surface_strides(const struct pan_image_layout *layout, unsigned l,
176 int32_t *row_stride, int32_t *surf_stride)
177 {
178 const struct pan_image_slice_layout *slice = &layout->slices[l];
179
180 if (drm_is_afbc(layout->modifier)) {
181 /* Pre v7 don't have a row stride field. This field is
182 * repurposed as a Y offset which we don't use */
183 *row_stride = PAN_ARCH < 7 ? 0 : slice->row_stride;
184 *surf_stride = slice->afbc.surface_stride;
185 } else {
186 *row_stride = slice->row_stride;
187 *surf_stride = slice->surface_stride;
188 }
189 }
190
191 static mali_ptr
panfrost_get_surface_pointer(const struct pan_image_layout * layout,enum mali_texture_dimension dim,mali_ptr base,unsigned l,unsigned i,unsigned s)192 panfrost_get_surface_pointer(const struct pan_image_layout *layout,
193 enum mali_texture_dimension dim, mali_ptr base,
194 unsigned l, unsigned i, unsigned s)
195 {
196 unsigned offset;
197
198 if (layout->dim == MALI_TEXTURE_DIMENSION_3D) {
199 assert(!s);
200 offset =
201 layout->slices[l].offset + i * panfrost_get_layer_stride(layout, l);
202 } else {
203 offset = panfrost_texture_offset(layout, l, i, s);
204 }
205
206 return base + offset;
207 }
208
209 #if PAN_ARCH <= 7
210 static void
panfrost_emit_surface_with_stride(mali_ptr plane,int32_t row_stride,int32_t surface_stride,void ** payload)211 panfrost_emit_surface_with_stride(mali_ptr plane, int32_t row_stride,
212 int32_t surface_stride, void **payload)
213 {
214 pan_pack(*payload, SURFACE_WITH_STRIDE, cfg) {
215 cfg.pointer = plane;
216 cfg.row_stride = row_stride;
217 cfg.surface_stride = surface_stride;
218 }
219 *payload += pan_size(SURFACE_WITH_STRIDE);
220 }
221 #endif
222
223 #if PAN_ARCH == 7
224 static void
panfrost_emit_multiplanar_surface(mali_ptr planes[MAX_IMAGE_PLANES],int32_t row_strides[MAX_IMAGE_PLANES],void ** payload)225 panfrost_emit_multiplanar_surface(mali_ptr planes[MAX_IMAGE_PLANES],
226 int32_t row_strides[MAX_IMAGE_PLANES],
227 void **payload)
228 {
229 assert(row_strides[2] == 0 || row_strides[1] == row_strides[2]);
230
231 pan_pack(*payload, MULTIPLANAR_SURFACE, cfg) {
232 cfg.plane_0_pointer = planes[0];
233 cfg.plane_0_row_stride = row_strides[0];
234 cfg.plane_1_2_row_stride = row_strides[1];
235 cfg.plane_1_pointer = planes[1];
236 cfg.plane_2_pointer = planes[2];
237 }
238 *payload += pan_size(MULTIPLANAR_SURFACE);
239 }
240 #endif
241
242 #if PAN_ARCH >= 9
243
244 /* clang-format off */
245 #define CLUMP_FMT(pipe, mali) [PIPE_FORMAT_ ## pipe] = MALI_CLUMP_FORMAT_ ## mali
246 static enum mali_clump_format special_clump_formats[PIPE_FORMAT_COUNT] = {
247 CLUMP_FMT(X32_S8X24_UINT, X32S8X24),
248 CLUMP_FMT(X24S8_UINT, X24S8),
249 CLUMP_FMT(S8X24_UINT, S8X24),
250 CLUMP_FMT(S8_UINT, S8),
251 CLUMP_FMT(L4A4_UNORM, L4A4),
252 CLUMP_FMT(L8A8_UNORM, L8A8),
253 CLUMP_FMT(L8A8_UINT, L8A8),
254 CLUMP_FMT(L8A8_SINT, L8A8),
255 CLUMP_FMT(A8_UNORM, A8),
256 CLUMP_FMT(A8_UINT, A8),
257 CLUMP_FMT(A8_SINT, A8),
258 CLUMP_FMT(ETC1_RGB8, ETC2_RGB8),
259 CLUMP_FMT(ETC2_RGB8, ETC2_RGB8),
260 CLUMP_FMT(ETC2_SRGB8, ETC2_RGB8),
261 CLUMP_FMT(ETC2_RGB8A1, ETC2_RGB8A1),
262 CLUMP_FMT(ETC2_SRGB8A1, ETC2_RGB8A1),
263 CLUMP_FMT(ETC2_RGBA8, ETC2_RGBA8),
264 CLUMP_FMT(ETC2_SRGBA8, ETC2_RGBA8),
265 CLUMP_FMT(ETC2_R11_UNORM, ETC2_R11_UNORM),
266 CLUMP_FMT(ETC2_R11_SNORM, ETC2_R11_SNORM),
267 CLUMP_FMT(ETC2_RG11_UNORM, ETC2_RG11_UNORM),
268 CLUMP_FMT(ETC2_RG11_SNORM, ETC2_RG11_SNORM),
269 CLUMP_FMT(DXT1_RGB, BC1_UNORM),
270 CLUMP_FMT(DXT1_RGBA, BC1_UNORM),
271 CLUMP_FMT(DXT1_SRGB, BC1_UNORM),
272 CLUMP_FMT(DXT1_SRGBA, BC1_UNORM),
273 CLUMP_FMT(DXT3_RGBA, BC2_UNORM),
274 CLUMP_FMT(DXT3_SRGBA, BC2_UNORM),
275 CLUMP_FMT(DXT5_RGBA, BC3_UNORM),
276 CLUMP_FMT(DXT5_SRGBA, BC3_UNORM),
277 CLUMP_FMT(RGTC1_UNORM, BC4_UNORM),
278 CLUMP_FMT(RGTC1_SNORM, BC4_SNORM),
279 CLUMP_FMT(RGTC2_UNORM, BC5_UNORM),
280 CLUMP_FMT(RGTC2_SNORM, BC5_SNORM),
281 CLUMP_FMT(BPTC_RGB_FLOAT, BC6H_SF16),
282 CLUMP_FMT(BPTC_RGB_UFLOAT, BC6H_UF16),
283 CLUMP_FMT(BPTC_RGBA_UNORM, BC7_UNORM),
284 CLUMP_FMT(BPTC_SRGBA, BC7_UNORM),
285 };
286 #undef CLUMP_FMT
287 /* clang-format on */
288
289 static enum mali_clump_format
panfrost_clump_format(enum pipe_format format)290 panfrost_clump_format(enum pipe_format format)
291 {
292 /* First, try a special clump format. Note that the 0 encoding is for a
293 * raw clump format, which will never be in the special table.
294 */
295 if (special_clump_formats[format])
296 return special_clump_formats[format];
297
298 /* Else, it's a raw format. Raw formats must not be compressed. */
299 assert(!util_format_is_compressed(format));
300
301 /* YUV-sampling has special cases */
302 if (panfrost_format_is_yuv(format)) {
303 switch (format) {
304 case PIPE_FORMAT_R8G8_R8B8_UNORM:
305 case PIPE_FORMAT_G8R8_B8R8_UNORM:
306 case PIPE_FORMAT_R8B8_R8G8_UNORM:
307 case PIPE_FORMAT_B8R8_G8R8_UNORM:
308 return MALI_CLUMP_FORMAT_Y8_UV8_422;
309 case PIPE_FORMAT_R8_G8B8_420_UNORM:
310 case PIPE_FORMAT_R8_B8G8_420_UNORM:
311 case PIPE_FORMAT_R8_G8_B8_420_UNORM:
312 case PIPE_FORMAT_R8_B8_G8_420_UNORM:
313 return MALI_CLUMP_FORMAT_Y8_UV8_420;
314 default:
315 unreachable("unhandled clump format");
316 }
317 }
318
319 /* Select the appropriate raw format. */
320 switch (util_format_get_blocksize(format)) {
321 case 1:
322 return MALI_CLUMP_FORMAT_RAW8;
323 case 2:
324 return MALI_CLUMP_FORMAT_RAW16;
325 case 3:
326 return MALI_CLUMP_FORMAT_RAW24;
327 case 4:
328 return MALI_CLUMP_FORMAT_RAW32;
329 case 6:
330 return MALI_CLUMP_FORMAT_RAW48;
331 case 8:
332 return MALI_CLUMP_FORMAT_RAW64;
333 case 12:
334 return MALI_CLUMP_FORMAT_RAW96;
335 case 16:
336 return MALI_CLUMP_FORMAT_RAW128;
337 default:
338 unreachable("Invalid bpp");
339 }
340 }
341
342 static enum mali_afbc_superblock_size
translate_superblock_size(uint64_t modifier)343 translate_superblock_size(uint64_t modifier)
344 {
345 assert(drm_is_afbc(modifier));
346
347 switch (modifier & AFBC_FORMAT_MOD_BLOCK_SIZE_MASK) {
348 case AFBC_FORMAT_MOD_BLOCK_SIZE_16x16:
349 return MALI_AFBC_SUPERBLOCK_SIZE_16X16;
350 case AFBC_FORMAT_MOD_BLOCK_SIZE_32x8:
351 return MALI_AFBC_SUPERBLOCK_SIZE_32X8;
352 case AFBC_FORMAT_MOD_BLOCK_SIZE_64x4:
353 return MALI_AFBC_SUPERBLOCK_SIZE_64X4;
354 default:
355 unreachable("Invalid superblock size");
356 }
357 }
358
359 static void
panfrost_emit_plane(const struct pan_image_view * iview,int index,const struct pan_image_layout * layout,enum pipe_format format,mali_ptr pointer,unsigned level,int32_t row_stride,int32_t surface_stride,mali_ptr plane2_ptr,void ** payload)360 panfrost_emit_plane(const struct pan_image_view *iview, int index,
361 const struct pan_image_layout *layout,
362 enum pipe_format format, mali_ptr pointer, unsigned level,
363 int32_t row_stride, int32_t surface_stride,
364 mali_ptr plane2_ptr, void **payload)
365 {
366 const struct util_format_description *desc =
367 util_format_description(format);
368
369 assert(row_stride >= 0 && surface_stride >= 0 && "negative stride");
370
371 bool afbc = drm_is_afbc(layout->modifier);
372 bool afrc = drm_is_afrc(layout->modifier);
373 // TODO: this isn't technically guaranteed to be YUV, but it is in practice.
374 bool is_3_planar_yuv = desc->layout == UTIL_FORMAT_LAYOUT_PLANAR3;
375
376 pan_pack(*payload, PLANE, cfg) {
377 cfg.pointer = pointer;
378 cfg.row_stride = row_stride;
379 cfg.size = layout->data_size - layout->slices[level].offset;
380
381 if (is_3_planar_yuv) {
382 cfg.two_plane_yuv_chroma.secondary_pointer = plane2_ptr;
383 } else if (!panfrost_format_is_yuv(layout->format)) {
384 cfg.slice_stride = layout->nr_samples
385 ? surface_stride
386 : panfrost_get_layer_stride(layout, level);
387 }
388
389 if (desc->layout == UTIL_FORMAT_LAYOUT_ASTC) {
390 assert(!afbc);
391 assert(!afrc);
392
393 if (desc->block.depth > 1) {
394 cfg.plane_type = MALI_PLANE_TYPE_ASTC_3D;
395 cfg.astc._3d.block_width = panfrost_astc_dim_3d(desc->block.width);
396 cfg.astc._3d.block_height =
397 panfrost_astc_dim_3d(desc->block.height);
398 cfg.astc._3d.block_depth = panfrost_astc_dim_3d(desc->block.depth);
399 } else {
400 cfg.plane_type = MALI_PLANE_TYPE_ASTC_2D;
401 cfg.astc._2d.block_width = panfrost_astc_dim_2d(desc->block.width);
402 cfg.astc._2d.block_height =
403 panfrost_astc_dim_2d(desc->block.height);
404 }
405
406 bool srgb = (desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB);
407
408 /* Mesa does not advertise _HDR formats yet */
409 cfg.astc.decode_hdr = false;
410
411 /* sRGB formats decode to RGBA8 sRGB, which is narrow.
412 *
413 * Non-sRGB formats decode to RGBA16F which is wide except if decode
414 * precision is set to GL_RGBA8 for that texture.
415 */
416 cfg.astc.decode_wide = !srgb && !iview->astc.narrow;
417 } else if (afbc) {
418 cfg.plane_type = MALI_PLANE_TYPE_AFBC;
419 cfg.afbc.superblock_size = translate_superblock_size(layout->modifier);
420 cfg.afbc.ytr = (layout->modifier & AFBC_FORMAT_MOD_YTR);
421 cfg.afbc.tiled_header = (layout->modifier & AFBC_FORMAT_MOD_TILED);
422 cfg.afbc.prefetch = true;
423 cfg.afbc.compression_mode = GENX(pan_afbc_compression_mode)(format);
424 cfg.afbc.header_stride = layout->slices[level].afbc.header_size;
425 } else if (afrc) {
426 #if PAN_ARCH >= 10
427 struct pan_afrc_format_info finfo =
428 panfrost_afrc_get_format_info(format);
429
430 cfg.plane_type = MALI_PLANE_TYPE_AFRC;
431 cfg.afrc.block_size =
432 GENX(pan_afrc_block_size)(layout->modifier, index);
433 cfg.afrc.format =
434 GENX(pan_afrc_format)(finfo, layout->modifier, index);
435 #endif
436 } else {
437 cfg.plane_type = is_3_planar_yuv ? MALI_PLANE_TYPE_CHROMA_2P
438 : MALI_PLANE_TYPE_GENERIC;
439 cfg.clump_format = panfrost_clump_format(format);
440 }
441
442 if (!afbc && !afrc) {
443 if (layout->modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED)
444 cfg.clump_ordering = MALI_CLUMP_ORDERING_TILED_U_INTERLEAVED;
445 else
446 cfg.clump_ordering = MALI_CLUMP_ORDERING_LINEAR;
447 }
448 }
449 *payload += pan_size(PLANE);
450 }
451 #endif
452
453 static void
panfrost_emit_surface(const struct pan_image_view * iview,unsigned level,unsigned index,unsigned sample,enum pipe_format format,void ** payload)454 panfrost_emit_surface(const struct pan_image_view *iview, unsigned level,
455 unsigned index, unsigned sample,
456 enum pipe_format format, void **payload)
457 {
458 ASSERTED const struct util_format_description *desc =
459 util_format_description(format);
460
461 const struct pan_image_layout *layouts[MAX_IMAGE_PLANES] = {0};
462 mali_ptr plane_ptrs[MAX_IMAGE_PLANES] = {0};
463 int32_t row_strides[MAX_IMAGE_PLANES] = {0};
464 int32_t surface_strides[MAX_IMAGE_PLANES] = {0};
465
466 for (int i = 0; i < MAX_IMAGE_PLANES; i++) {
467 const struct pan_image *base_image = pan_image_view_get_plane(iview, i);
468
469 if (!base_image) {
470 /* Every texture should have at least one plane. */
471 assert(i > 0);
472 break;
473 }
474
475 mali_ptr base = base_image->data.base + base_image->data.offset;
476
477 if (iview->buf.size) {
478 assert(iview->dim == MALI_TEXTURE_DIMENSION_1D);
479 base += iview->buf.offset;
480 }
481
482 layouts[i] = &pan_image_view_get_plane(iview, i)->layout;
483
484 /* v4 does not support compression */
485 assert(PAN_ARCH >= 5 || !drm_is_afbc(layouts[i]->modifier));
486 assert(PAN_ARCH >= 5 || desc->layout != UTIL_FORMAT_LAYOUT_ASTC);
487
488 /* panfrost_compression_tag() wants the dimension of the resource, not the
489 * one of the image view (those might differ).
490 */
491 unsigned tag =
492 panfrost_compression_tag(desc, layouts[i]->dim, layouts[i]->modifier);
493
494 plane_ptrs[i] = panfrost_get_surface_pointer(
495 layouts[i], iview->dim, base | tag, level, index, sample);
496 panfrost_get_surface_strides(layouts[i], level, &row_strides[i],
497 &surface_strides[i]);
498 }
499
500 #if PAN_ARCH >= 9
501 if (panfrost_format_is_yuv(format)) {
502 for (int i = 0; i < MAX_IMAGE_PLANES; i++) {
503 /* 3-plane YUV is submitted using two PLANE descriptors, where the
504 * second one is of type CHROMA_2P */
505 if (i > 1)
506 break;
507
508 if (plane_ptrs[i] == 0)
509 break;
510
511 /* 3-plane YUV requires equal stride for both chroma planes */
512 assert(row_strides[2] == 0 || row_strides[1] == row_strides[2]);
513
514 panfrost_emit_plane(iview, i, layouts[i], format, plane_ptrs[i], level,
515 row_strides[i], surface_strides[i], plane_ptrs[2],
516 payload);
517 }
518 } else {
519 panfrost_emit_plane(iview, 0, layouts[0], format, plane_ptrs[0], level,
520 row_strides[0], surface_strides[0], 0, payload);
521 }
522 return;
523 #endif
524
525 #if PAN_ARCH <= 7
526 #if PAN_ARCH == 7
527 if (panfrost_format_is_yuv(format)) {
528 panfrost_emit_multiplanar_surface(plane_ptrs, row_strides, payload);
529 return;
530 }
531 #endif
532 panfrost_emit_surface_with_stride(plane_ptrs[0], row_strides[0],
533 surface_strides[0], payload);
534 #endif
535 }
536
537 static void
panfrost_emit_texture_payload(const struct pan_image_view * iview,enum pipe_format format,void * payload)538 panfrost_emit_texture_payload(const struct pan_image_view *iview,
539 enum pipe_format format, void *payload)
540 {
541 unsigned nr_samples =
542 PAN_ARCH <= 7 ? pan_image_view_get_nr_samples(iview) : 1;
543
544 /* Inject the addresses in, interleaving array indices, mip levels,
545 * cube faces, and strides in that order. On Bifrost and older, each
546 * sample had its own surface descriptor; on Valhall, they are fused
547 * into a single plane descriptor.
548 */
549
550 #if PAN_ARCH >= 7
551 /* V7 and later treats faces as extra layers */
552 for (int layer = iview->first_layer; layer <= iview->last_layer; ++layer) {
553 for (int sample = 0; sample < nr_samples; ++sample) {
554 for (int level = iview->first_level; level <= iview->last_level; ++level) {
555 panfrost_emit_surface(iview, level, layer, sample,
556 format, &payload);
557 }
558 }
559 }
560 #else
561 unsigned first_layer = iview->first_layer, last_layer = iview->last_layer;
562 unsigned face_count = 1;
563
564 if (iview->dim == MALI_TEXTURE_DIMENSION_CUBE) {
565 first_layer /= 6;
566 last_layer /= 6;
567 face_count = 6;
568 }
569
570 /* V6 and earlier has a different memory-layout */
571 for (int layer = first_layer; layer <= last_layer; ++layer) {
572 for (int level = iview->first_level; level <= iview->last_level; ++level) {
573 /* order of face and sample doesn't matter; we can only have multiple
574 * of one or the other (no support for multisampled cubemaps)
575 */
576 for (int face = 0; face < face_count; ++face) {
577 for (int sample = 0; sample < nr_samples; ++sample) {
578 panfrost_emit_surface(iview, level, (face_count * layer) + face,
579 sample, format, &payload);
580 }
581 }
582 }
583 }
584 #endif
585 }
586
587 #if PAN_ARCH <= 7
588 /* Map modifiers to mali_texture_layout for packing in a texture descriptor */
589
590 static enum mali_texture_layout
panfrost_modifier_to_layout(uint64_t modifier)591 panfrost_modifier_to_layout(uint64_t modifier)
592 {
593 if (drm_is_afbc(modifier))
594 return MALI_TEXTURE_LAYOUT_AFBC;
595 else if (modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED)
596 return MALI_TEXTURE_LAYOUT_TILED;
597 else if (modifier == DRM_FORMAT_MOD_LINEAR)
598 return MALI_TEXTURE_LAYOUT_LINEAR;
599 else
600 unreachable("Invalid modifer");
601 }
602 #endif
603
604 /*
605 * Generates a texture descriptor. Ideally, descriptors are immutable after the
606 * texture is created, so we can keep these hanging around in GPU memory in a
607 * dedicated BO and not have to worry. In practice there are some minor gotchas
608 * with this (the driver sometimes will change the format of a texture on the
609 * fly for compression) but it's fast enough to just regenerate the descriptor
610 * in those cases, rather than monkeypatching at drawtime. A texture descriptor
611 * consists of a 32-byte header followed by pointers.
612 */
613 void
GENX(panfrost_new_texture)614 GENX(panfrost_new_texture)(const struct pan_image_view *iview, void *out,
615 const struct panfrost_ptr *payload)
616 {
617 const struct pan_image *base_image = pan_image_view_get_plane(iview, 0);
618 const struct pan_image_layout *layout = &base_image->layout;
619 enum pipe_format format = iview->format;
620 const struct util_format_description *desc = util_format_description(format);
621 uint32_t mali_format = GENX(panfrost_format_from_pipe_format)(format)->hw;
622 unsigned char swizzle[4];
623
624 if (desc->layout == UTIL_FORMAT_LAYOUT_ASTC && iview->astc.narrow &&
625 desc->colorspace != UTIL_FORMAT_COLORSPACE_SRGB) {
626 mali_format = MALI_PACK_FMT(RGBA8_UNORM, RGBA, L);
627 }
628
629 if (PAN_ARCH >= 7 && util_format_is_depth_or_stencil(format)) {
630 /* v7+ doesn't have an _RRRR component order, combine the
631 * user swizzle with a .XXXX swizzle to emulate that.
632 */
633 static const unsigned char replicate_x[4] = {
634 PIPE_SWIZZLE_X,
635 PIPE_SWIZZLE_X,
636 PIPE_SWIZZLE_X,
637 PIPE_SWIZZLE_X,
638 };
639
640 util_format_compose_swizzles(replicate_x, iview->swizzle, swizzle);
641 } else if ((PAN_ARCH == 7 || PAN_ARCH == 10) &&
642 !panfrost_format_is_yuv(format)) {
643 #if PAN_ARCH == 7 || PAN_ARCH >= 10
644 /* v7 (only) restricts component orders when AFBC is in use.
645 * Rather than restrict AFBC, we use an allowed component order
646 * with an invertible swizzle composed.
647 * v10 has the same restriction, but on AFRC formats.
648 */
649 enum mali_rgb_component_order orig = mali_format & BITFIELD_MASK(12);
650 struct pan_decomposed_swizzle decomposed =
651 GENX(pan_decompose_swizzle)(orig);
652
653 /* Apply the new component order */
654 mali_format = (mali_format & ~orig) | decomposed.pre;
655
656 /* Compose the new swizzle */
657 util_format_compose_swizzles(decomposed.post, iview->swizzle, swizzle);
658 #endif
659 } else {
660 STATIC_ASSERT(sizeof(swizzle) == sizeof(iview->swizzle));
661 memcpy(swizzle, iview->swizzle, sizeof(swizzle));
662 }
663
664 panfrost_emit_texture_payload(iview, format, payload->cpu);
665
666 unsigned array_size = iview->last_layer - iview->first_layer + 1;
667
668 /* If this is a cubemap, we expect the number of layers to be a multiple
669 * of 6.
670 */
671 if (iview->dim == MALI_TEXTURE_DIMENSION_CUBE) {
672 assert(array_size % 6 == 0);
673 array_size /= 6;
674 }
675
676 /* Multiplanar YUV textures require 2 surface descriptors. */
677 if (panfrost_format_is_yuv(iview->format) && PAN_ARCH >= 9 &&
678 pan_image_view_get_plane(iview, 1) != NULL)
679 array_size *= 2;
680
681 unsigned width, height, depth;
682
683 if (iview->buf.size) {
684 assert(iview->dim == MALI_TEXTURE_DIMENSION_1D);
685 assert(!iview->first_level && !iview->last_level);
686 assert(!iview->first_layer && !iview->last_layer);
687 assert(layout->nr_samples == 1);
688 assert(layout->height == 1 && layout->depth == 1);
689 assert(iview->buf.offset + iview->buf.size <= layout->width);
690 width = iview->buf.size;
691 height = 1;
692 depth = 1;
693 } else {
694 width = u_minify(layout->width, iview->first_level);
695 height = u_minify(layout->height, iview->first_level);
696 depth = u_minify(layout->depth, iview->first_level);
697 if (util_format_is_compressed(layout->format) &&
698 !util_format_is_compressed(format)) {
699 width =
700 DIV_ROUND_UP(width, util_format_get_blockwidth(layout->format));
701 height =
702 DIV_ROUND_UP(height, util_format_get_blockheight(layout->format));
703 depth =
704 DIV_ROUND_UP(depth, util_format_get_blockdepth(layout->format));
705 assert(util_format_get_blockwidth(format) == 1);
706 assert(util_format_get_blockheight(format) == 1);
707 assert(util_format_get_blockheight(format) == 1);
708 assert(iview->last_level == iview->first_level);
709 }
710 }
711
712 pan_pack(out, TEXTURE, cfg) {
713 cfg.dimension = iview->dim;
714 cfg.format = mali_format;
715 cfg.width = width;
716 cfg.height = height;
717 if (iview->dim == MALI_TEXTURE_DIMENSION_3D)
718 cfg.depth = depth;
719 else
720 cfg.sample_count = layout->nr_samples;
721 cfg.swizzle = panfrost_translate_swizzle_4(swizzle);
722 #if PAN_ARCH >= 9
723 cfg.texel_interleave = (layout->modifier != DRM_FORMAT_MOD_LINEAR) ||
724 util_format_is_compressed(format);
725 #else
726 cfg.texel_ordering = panfrost_modifier_to_layout(layout->modifier);
727 #endif
728 cfg.levels = iview->last_level - iview->first_level + 1;
729 cfg.array_size = array_size;
730
731 #if PAN_ARCH >= 6
732 cfg.surfaces = payload->gpu;
733
734 /* We specify API-level LOD clamps in the sampler descriptor
735 * and use these clamps simply for bounds checking.
736 */
737 cfg.minimum_lod = 0;
738 cfg.maximum_lod = cfg.levels - 1;
739 #endif
740 }
741 }
742
743 #if PAN_ARCH >= 9
744 enum mali_afbc_compression_mode
GENX(pan_afbc_compression_mode)745 GENX(pan_afbc_compression_mode)(enum pipe_format format)
746 {
747 /* There's a special case for texturing the stencil part from a combined
748 * depth/stencil texture, handle it separately.
749 */
750 if (format == PIPE_FORMAT_X24S8_UINT)
751 return MALI_AFBC_COMPRESSION_MODE_X24S8;
752
753 /* Otherwise, map canonical formats to the hardware enum. This only
754 * needs to handle the subset of formats returned by
755 * panfrost_afbc_format.
756 */
757 /* clang-format off */
758 switch (panfrost_afbc_format(PAN_ARCH, format)) {
759 case PAN_AFBC_MODE_R8: return MALI_AFBC_COMPRESSION_MODE_R8;
760 case PAN_AFBC_MODE_R8G8: return MALI_AFBC_COMPRESSION_MODE_R8G8;
761 case PAN_AFBC_MODE_R5G6B5: return MALI_AFBC_COMPRESSION_MODE_R5G6B5;
762 case PAN_AFBC_MODE_R4G4B4A4: return MALI_AFBC_COMPRESSION_MODE_R4G4B4A4;
763 case PAN_AFBC_MODE_R5G5B5A1: return MALI_AFBC_COMPRESSION_MODE_R5G5B5A1;
764 case PAN_AFBC_MODE_R8G8B8: return MALI_AFBC_COMPRESSION_MODE_R8G8B8;
765 case PAN_AFBC_MODE_R8G8B8A8: return MALI_AFBC_COMPRESSION_MODE_R8G8B8A8;
766 case PAN_AFBC_MODE_R10G10B10A2: return MALI_AFBC_COMPRESSION_MODE_R10G10B10A2;
767 case PAN_AFBC_MODE_R11G11B10: return MALI_AFBC_COMPRESSION_MODE_R11G11B10;
768 case PAN_AFBC_MODE_S8: return MALI_AFBC_COMPRESSION_MODE_S8;
769 case PAN_AFBC_MODE_INVALID: unreachable("Invalid AFBC format");
770 }
771 /* clang-format on */
772
773 unreachable("all AFBC formats handled");
774 }
775 #endif
776
777 #if PAN_ARCH >= 10
778 enum mali_afrc_format
GENX(pan_afrc_format)779 GENX(pan_afrc_format)(struct pan_afrc_format_info info, uint64_t modifier,
780 unsigned plane)
781 {
782 bool scan = panfrost_afrc_is_scan(modifier);
783
784 assert(info.bpc == 8 || info.bpc == 10);
785 assert(info.num_comps > 0 && info.num_comps <= 4);
786
787 switch (info.ichange_fmt) {
788 case PAN_AFRC_ICHANGE_FORMAT_RAW:
789 assert(plane == 0);
790
791 if (info.bpc == 8)
792 return (scan ? MALI_AFRC_FORMAT_R8_SCAN : MALI_AFRC_FORMAT_R8_ROT) +
793 (info.num_comps - 1);
794
795 assert(info.num_comps == 4);
796 return (scan ? MALI_AFRC_FORMAT_R10G10B10A10_SCAN
797 : MALI_AFRC_FORMAT_R10G10B10A10_ROT);
798
799 case PAN_AFRC_ICHANGE_FORMAT_YUV444:
800 if (info.bpc == 8) {
801 if (plane == 0 || info.num_planes == 3)
802 return (scan ? MALI_AFRC_FORMAT_R8_444_SCAN
803 : MALI_AFRC_FORMAT_R8_444_ROT);
804
805 return (scan ? MALI_AFRC_FORMAT_R8G8_444_SCAN
806 : MALI_AFRC_FORMAT_R8G8_444_ROT);
807 }
808
809 assert(info.num_planes == 3);
810 return (scan ? MALI_AFRC_FORMAT_R10_444_SCAN
811 : MALI_AFRC_FORMAT_R10_444_ROT);
812
813 case PAN_AFRC_ICHANGE_FORMAT_YUV422:
814 if (info.bpc == 8) {
815 if (plane == 0 || info.num_planes == 3)
816 return (scan ? MALI_AFRC_FORMAT_R8_422_SCAN
817 : MALI_AFRC_FORMAT_R8_422_ROT);
818
819 return (scan ? MALI_AFRC_FORMAT_R8G8_422_SCAN
820 : MALI_AFRC_FORMAT_R8G8_422_ROT);
821 }
822
823 if (plane == 0 || info.num_planes == 3)
824 return (scan ? MALI_AFRC_FORMAT_R10_422_SCAN
825 : MALI_AFRC_FORMAT_R10_422_ROT);
826
827 return (scan ? MALI_AFRC_FORMAT_R10G10_422_SCAN
828 : MALI_AFRC_FORMAT_R10G10_422_ROT);
829
830 case PAN_AFRC_ICHANGE_FORMAT_YUV420:
831 if (info.bpc == 8) {
832 if (plane == 0 || info.num_planes == 3)
833 return (scan ? MALI_AFRC_FORMAT_R8_420_SCAN
834 : MALI_AFRC_FORMAT_R8_420_ROT);
835
836 return (scan ? MALI_AFRC_FORMAT_R8G8_420_SCAN
837 : MALI_AFRC_FORMAT_R8G8_420_ROT);
838 }
839
840 if (plane == 0 || info.num_planes == 3)
841 return (scan ? MALI_AFRC_FORMAT_R10_420_SCAN
842 : MALI_AFRC_FORMAT_R10_420_ROT);
843
844 return (scan ? MALI_AFRC_FORMAT_R10G10_420_SCAN
845 : MALI_AFRC_FORMAT_R10G10_420_ROT);
846
847 default:
848 return MALI_AFRC_FORMAT_INVALID;
849 }
850 }
851
852 enum mali_afrc_block_size
GENX(pan_afrc_block_size)853 GENX(pan_afrc_block_size)(uint64_t modifier, unsigned index)
854 {
855 /* Clump size flag for planes 1 and 2 is shifted by 4 bits */
856 unsigned shift = index == 0 ? 0 : 4;
857 uint64_t flag = (modifier >> shift) & AFRC_FORMAT_MOD_CU_SIZE_MASK;
858
859 /* clang-format off */
860 switch (flag) {
861 case AFRC_FORMAT_MOD_CU_SIZE_16: return MALI_AFRC_BLOCK_SIZE_16;
862 case AFRC_FORMAT_MOD_CU_SIZE_24: return MALI_AFRC_BLOCK_SIZE_24;
863 case AFRC_FORMAT_MOD_CU_SIZE_32: return MALI_AFRC_BLOCK_SIZE_32;
864 default: unreachable("invalid code unit size");
865 }
866 /* clang-format on */
867 }
868 #endif
869