1 /*
2 * Copyright © 2014-2017 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 /** @file v3d_tiling.c
25 *
26 * Handles information about the V3D tiling formats, and loading and storing
27 * from them.
28 */
29
30 #include <stdint.h>
31 #include "util/box.h"
32 #include "v3d_tiling.h"
33 #include "broadcom/common/v3d_cpu_tiling.h"
34
35 /** Return the width in pixels of a 64-byte microtile. */
36 uint32_t
v3d_utile_width(int cpp)37 v3d_utile_width(int cpp)
38 {
39 switch (cpp) {
40 case 1:
41 case 2:
42 return 8;
43 case 4:
44 case 8:
45 return 4;
46 case 16:
47 return 2;
48 default:
49 unreachable("unknown cpp");
50 }
51 }
52
53 /** Return the height in pixels of a 64-byte microtile. */
54 uint32_t
v3d_utile_height(int cpp)55 v3d_utile_height(int cpp)
56 {
57 switch (cpp) {
58 case 1:
59 return 8;
60 case 2:
61 case 4:
62 return 4;
63 case 8:
64 case 16:
65 return 2;
66 default:
67 unreachable("unknown cpp");
68 }
69 }
70
71 /**
72 * Returns the byte address for a given pixel within a utile.
73 *
74 * Utiles are 64b blocks of pixels in raster order, with 32bpp being a 4x4
75 * arrangement.
76 */
77 static inline uint32_t
v3d_get_utile_pixel_offset(uint32_t cpp,uint32_t x,uint32_t y)78 v3d_get_utile_pixel_offset(uint32_t cpp, uint32_t x, uint32_t y)
79 {
80 uint32_t utile_w = v3d_utile_width(cpp);
81
82 assert(x < utile_w && y < v3d_utile_height(cpp));
83
84 return x * cpp + y * utile_w * cpp;
85 }
86
87 /**
88 * Returns the byte offset for a given pixel in a LINEARTILE layout.
89 *
90 * LINEARTILE is a single line of utiles in either the X or Y direction.
91 */
92 static inline uint32_t
v3d_get_lt_pixel_offset(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y)93 v3d_get_lt_pixel_offset(uint32_t cpp, uint32_t image_h, uint32_t x, uint32_t y)
94 {
95 uint32_t utile_w = v3d_utile_width(cpp);
96 uint32_t utile_h = v3d_utile_height(cpp);
97 uint32_t utile_index_x = x / utile_w;
98 uint32_t utile_index_y = y / utile_h;
99
100 assert(utile_index_x == 0 || utile_index_y == 0);
101
102 return (64 * (utile_index_x + utile_index_y) +
103 v3d_get_utile_pixel_offset(cpp,
104 x & (utile_w - 1),
105 y & (utile_h - 1)));
106 }
107
108 /**
109 * Returns the byte offset for a given pixel in a UBLINEAR layout.
110 *
111 * UBLINEAR is the layout where pixels are arranged in UIF blocks (2x2
112 * utiles), and the UIF blocks are in 1 or 2 columns in raster order.
113 */
114 static inline uint32_t
v3d_get_ublinear_pixel_offset(uint32_t cpp,uint32_t x,uint32_t y,int ublinear_number)115 v3d_get_ublinear_pixel_offset(uint32_t cpp, uint32_t x, uint32_t y,
116 int ublinear_number)
117 {
118 uint32_t utile_w = v3d_utile_width(cpp);
119 uint32_t utile_h = v3d_utile_height(cpp);
120 uint32_t ub_w = utile_w * 2;
121 uint32_t ub_h = utile_h * 2;
122 uint32_t ub_x = x / ub_w;
123 uint32_t ub_y = y / ub_h;
124
125 return (256 * (ub_y * ublinear_number +
126 ub_x) +
127 ((x & utile_w) ? 64 : 0) +
128 ((y & utile_h) ? 128 : 0) +
129 + v3d_get_utile_pixel_offset(cpp,
130 x & (utile_w - 1),
131 y & (utile_h - 1)));
132 }
133
134 static inline uint32_t
v3d_get_ublinear_2_column_pixel_offset(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y)135 v3d_get_ublinear_2_column_pixel_offset(uint32_t cpp, uint32_t image_h,
136 uint32_t x, uint32_t y)
137 {
138 return v3d_get_ublinear_pixel_offset(cpp, x, y, 2);
139 }
140
141 static inline uint32_t
v3d_get_ublinear_1_column_pixel_offset(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y)142 v3d_get_ublinear_1_column_pixel_offset(uint32_t cpp, uint32_t image_h,
143 uint32_t x, uint32_t y)
144 {
145 return v3d_get_ublinear_pixel_offset(cpp, x, y, 1);
146 }
147
148 /**
149 * Returns the byte offset for a given pixel in a UIF layout.
150 *
151 * UIF is the general V3D tiling layout shared across 3D, media, and scanout.
152 * It stores pixels in UIF blocks (2x2 utiles), and UIF blocks are stored in
153 * 4x4 groups, and those 4x4 groups are then stored in raster order.
154 */
155 static inline uint32_t
v3d_get_uif_pixel_offset(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y,bool do_xor)156 v3d_get_uif_pixel_offset(uint32_t cpp, uint32_t image_h, uint32_t x, uint32_t y,
157 bool do_xor)
158 {
159 uint32_t utile_w = v3d_utile_width(cpp);
160 uint32_t utile_h = v3d_utile_height(cpp);
161 uint32_t mb_width = utile_w * 2;
162 uint32_t mb_height = utile_h * 2;
163 uint32_t log2_mb_width = ffs(mb_width) - 1;
164 uint32_t log2_mb_height = ffs(mb_height) - 1;
165
166 /* Macroblock X, y */
167 uint32_t mb_x = x >> log2_mb_width;
168 uint32_t mb_y = y >> log2_mb_height;
169 /* X, y within the macroblock */
170 uint32_t mb_pixel_x = x - (mb_x << log2_mb_width);
171 uint32_t mb_pixel_y = y - (mb_y << log2_mb_height);
172
173 if (do_xor && (mb_x / 4) & 1)
174 mb_y ^= 0x10;
175
176 uint32_t mb_h = align(image_h, 1 << log2_mb_height) >> log2_mb_height;
177 uint32_t mb_id = ((mb_x / 4) * ((mb_h - 1) * 4)) + mb_x + mb_y * 4;
178
179 uint32_t mb_base_addr = mb_id * 256;
180
181 bool top = mb_pixel_y < utile_h;
182 bool left = mb_pixel_x < utile_w;
183
184 /* Docs have this in pixels, we do bytes here. */
185 uint32_t mb_tile_offset = (!top * 128 + !left * 64);
186
187 uint32_t utile_x = mb_pixel_x & (utile_w - 1);
188 uint32_t utile_y = mb_pixel_y & (utile_h - 1);
189
190 uint32_t mb_pixel_address = (mb_base_addr +
191 mb_tile_offset +
192 v3d_get_utile_pixel_offset(cpp,
193 utile_x,
194 utile_y));
195
196 return mb_pixel_address;
197 }
198
199 static inline uint32_t
v3d_get_uif_xor_pixel_offset(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y)200 v3d_get_uif_xor_pixel_offset(uint32_t cpp, uint32_t image_h,
201 uint32_t x, uint32_t y)
202 {
203 return v3d_get_uif_pixel_offset(cpp, image_h, x, y, true);
204 }
205
206 static inline uint32_t
v3d_get_uif_no_xor_pixel_offset(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y)207 v3d_get_uif_no_xor_pixel_offset(uint32_t cpp, uint32_t image_h,
208 uint32_t x, uint32_t y)
209 {
210 return v3d_get_uif_pixel_offset(cpp, image_h, x, y, false);
211 }
212
213 /* Loads/stores non-utile-aligned boxes by walking over the destination
214 * rectangle, computing the address on the GPU, and storing/loading a pixel at
215 * a time.
216 */
217 static inline void
v3d_move_pixels_unaligned(void * gpu,uint32_t gpu_stride,void * cpu,uint32_t cpu_stride,int cpp,uint32_t image_h,const struct pipe_box * box,uint32_t (* get_pixel_offset)(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y),bool is_load)218 v3d_move_pixels_unaligned(void *gpu, uint32_t gpu_stride,
219 void *cpu, uint32_t cpu_stride,
220 int cpp, uint32_t image_h,
221 const struct pipe_box *box,
222 uint32_t (*get_pixel_offset)(uint32_t cpp,
223 uint32_t image_h,
224 uint32_t x, uint32_t y),
225 bool is_load)
226 {
227 for (uint32_t y = 0; y < box->height; y++) {
228 void *cpu_row = cpu + y * cpu_stride;
229
230 for (int x = 0; x < box->width; x++) {
231 uint32_t pixel_offset = get_pixel_offset(cpp, image_h,
232 box->x + x,
233 box->y + y);
234
235 if (false) {
236 fprintf(stderr, "%3d,%3d -> %d\n",
237 box->x + x, box->y + y,
238 pixel_offset);
239 }
240
241 if (is_load) {
242 memcpy(cpu_row + x * cpp,
243 gpu + pixel_offset,
244 cpp);
245 } else {
246 memcpy(gpu + pixel_offset,
247 cpu_row + x * cpp,
248 cpp);
249 }
250 }
251 }
252 }
253
254 /* Breaks the image down into utiles and calls either the fast whole-utile
255 * load/store functions, or the unaligned fallback case.
256 */
257 static inline void
v3d_move_pixels_general_percpp(void * gpu,uint32_t gpu_stride,void * cpu,uint32_t cpu_stride,int cpp,uint32_t image_h,const struct pipe_box * box,uint32_t (* get_pixel_offset)(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y),bool is_load)258 v3d_move_pixels_general_percpp(void *gpu, uint32_t gpu_stride,
259 void *cpu, uint32_t cpu_stride,
260 int cpp, uint32_t image_h,
261 const struct pipe_box *box,
262 uint32_t (*get_pixel_offset)(uint32_t cpp,
263 uint32_t image_h,
264 uint32_t x, uint32_t y),
265 bool is_load)
266 {
267 uint32_t utile_w = v3d_utile_width(cpp);
268 uint32_t utile_h = v3d_utile_height(cpp);
269 uint32_t utile_gpu_stride = utile_w * cpp;
270 uint32_t x1 = box->x;
271 uint32_t y1 = box->y;
272 uint32_t x2 = box->x + box->width;
273 uint32_t y2 = box->y + box->height;
274 uint32_t align_x1 = align(x1, utile_w);
275 uint32_t align_y1 = align(y1, utile_h);
276 uint32_t align_x2 = x2 & ~(utile_w - 1);
277 uint32_t align_y2 = y2 & ~(utile_h - 1);
278
279 /* Load/store all the whole utiles first. */
280 for (uint32_t y = align_y1; y < align_y2; y += utile_h) {
281 void *cpu_row = cpu + (y - box->y) * cpu_stride;
282
283 for (uint32_t x = align_x1; x < align_x2; x += utile_w) {
284 void *utile_gpu = (gpu +
285 get_pixel_offset(cpp, image_h, x, y));
286 void *utile_cpu = cpu_row + (x - box->x) * cpp;
287
288 if (is_load) {
289 v3d_load_utile(utile_cpu, cpu_stride,
290 utile_gpu, utile_gpu_stride);
291 } else {
292 v3d_store_utile(utile_gpu, utile_gpu_stride,
293 utile_cpu, cpu_stride);
294 }
295 }
296 }
297
298 /* If there were no aligned utiles in the middle, load/store the whole
299 * thing unaligned.
300 */
301 if (align_y2 <= align_y1 ||
302 align_x2 <= align_x1) {
303 v3d_move_pixels_unaligned(gpu, gpu_stride,
304 cpu, cpu_stride,
305 cpp, image_h,
306 box,
307 get_pixel_offset, is_load);
308 return;
309 }
310
311 /* Load/store the partial utiles. */
312 struct pipe_box partial_boxes[4] = {
313 /* Top */
314 {
315 .x = x1,
316 .width = x2 - x1,
317 .y = y1,
318 .height = align_y1 - y1,
319 },
320 /* Bottom */
321 {
322 .x = x1,
323 .width = x2 - x1,
324 .y = align_y2,
325 .height = y2 - align_y2,
326 },
327 /* Left */
328 {
329 .x = x1,
330 .width = align_x1 - x1,
331 .y = align_y1,
332 .height = align_y2 - align_y1,
333 },
334 /* Right */
335 {
336 .x = align_x2,
337 .width = x2 - align_x2,
338 .y = align_y1,
339 .height = align_y2 - align_y1,
340 },
341 };
342 for (int i = 0; i < ARRAY_SIZE(partial_boxes); i++) {
343 void *partial_cpu = (cpu +
344 (partial_boxes[i].y - y1) * cpu_stride +
345 (partial_boxes[i].x - x1) * cpp);
346
347 v3d_move_pixels_unaligned(gpu, gpu_stride,
348 partial_cpu, cpu_stride,
349 cpp, image_h,
350 &partial_boxes[i],
351 get_pixel_offset, is_load);
352 }
353 }
354
355 static inline void
v3d_move_pixels_general(void * gpu,uint32_t gpu_stride,void * cpu,uint32_t cpu_stride,int cpp,uint32_t image_h,const struct pipe_box * box,uint32_t (* get_pixel_offset)(uint32_t cpp,uint32_t image_h,uint32_t x,uint32_t y),bool is_load)356 v3d_move_pixels_general(void *gpu, uint32_t gpu_stride,
357 void *cpu, uint32_t cpu_stride,
358 int cpp, uint32_t image_h,
359 const struct pipe_box *box,
360 uint32_t (*get_pixel_offset)(uint32_t cpp,
361 uint32_t image_h,
362 uint32_t x, uint32_t y),
363 bool is_load)
364 {
365 switch (cpp) {
366 case 1:
367 v3d_move_pixels_general_percpp(gpu, gpu_stride,
368 cpu, cpu_stride,
369 1, image_h, box,
370 get_pixel_offset,
371 is_load);
372 break;
373 case 2:
374 v3d_move_pixels_general_percpp(gpu, gpu_stride,
375 cpu, cpu_stride,
376 2, image_h, box,
377 get_pixel_offset,
378 is_load);
379 break;
380 case 4:
381 v3d_move_pixels_general_percpp(gpu, gpu_stride,
382 cpu, cpu_stride,
383 4, image_h, box,
384 get_pixel_offset,
385 is_load);
386 break;
387 case 8:
388 v3d_move_pixels_general_percpp(gpu, gpu_stride,
389 cpu, cpu_stride,
390 8, image_h, box,
391 get_pixel_offset,
392 is_load);
393 break;
394 case 16:
395 v3d_move_pixels_general_percpp(gpu, gpu_stride,
396 cpu, cpu_stride,
397 16, image_h, box,
398 get_pixel_offset,
399 is_load);
400 break;
401 }
402 }
403
404 static inline void
v3d_move_tiled_image(void * gpu,uint32_t gpu_stride,void * cpu,uint32_t cpu_stride,enum v3d_tiling_mode tiling_format,int cpp,uint32_t image_h,const struct pipe_box * box,bool is_load)405 v3d_move_tiled_image(void *gpu, uint32_t gpu_stride,
406 void *cpu, uint32_t cpu_stride,
407 enum v3d_tiling_mode tiling_format,
408 int cpp,
409 uint32_t image_h,
410 const struct pipe_box *box,
411 bool is_load)
412 {
413 switch (tiling_format) {
414 case V3D_TILING_UIF_XOR:
415 v3d_move_pixels_general(gpu, gpu_stride,
416 cpu, cpu_stride,
417 cpp, image_h, box,
418 v3d_get_uif_xor_pixel_offset,
419 is_load);
420 break;
421 case V3D_TILING_UIF_NO_XOR:
422 v3d_move_pixels_general(gpu, gpu_stride,
423 cpu, cpu_stride,
424 cpp, image_h, box,
425 v3d_get_uif_no_xor_pixel_offset,
426 is_load);
427 break;
428 case V3D_TILING_UBLINEAR_2_COLUMN:
429 v3d_move_pixels_general(gpu, gpu_stride,
430 cpu, cpu_stride,
431 cpp, image_h, box,
432 v3d_get_ublinear_2_column_pixel_offset,
433 is_load);
434 break;
435 case V3D_TILING_UBLINEAR_1_COLUMN:
436 v3d_move_pixels_general(gpu, gpu_stride,
437 cpu, cpu_stride,
438 cpp, image_h, box,
439 v3d_get_ublinear_1_column_pixel_offset,
440 is_load);
441 break;
442 case V3D_TILING_LINEARTILE:
443 v3d_move_pixels_general(gpu, gpu_stride,
444 cpu, cpu_stride,
445 cpp, image_h, box,
446 v3d_get_lt_pixel_offset,
447 is_load);
448 break;
449 default:
450 unreachable("Unsupported tiling format");
451 break;
452 }
453 }
454
455 /**
456 * Loads pixel data from the start (microtile-aligned) box in \p src to the
457 * start of \p dst according to the given tiling format.
458 */
459 void
v3d_load_tiled_image(void * dst,uint32_t dst_stride,void * src,uint32_t src_stride,enum v3d_tiling_mode tiling_format,int cpp,uint32_t image_h,const struct pipe_box * box)460 v3d_load_tiled_image(void *dst, uint32_t dst_stride,
461 void *src, uint32_t src_stride,
462 enum v3d_tiling_mode tiling_format, int cpp,
463 uint32_t image_h,
464 const struct pipe_box *box)
465 {
466 v3d_move_tiled_image(src, src_stride,
467 dst, dst_stride,
468 tiling_format,
469 cpp,
470 image_h,
471 box,
472 true);
473 }
474
475 /**
476 * Stores pixel data from the start of \p src into a (microtile-aligned) box in
477 * \p dst according to the given tiling format.
478 */
479 void
v3d_store_tiled_image(void * dst,uint32_t dst_stride,void * src,uint32_t src_stride,enum v3d_tiling_mode tiling_format,int cpp,uint32_t image_h,const struct pipe_box * box)480 v3d_store_tiled_image(void *dst, uint32_t dst_stride,
481 void *src, uint32_t src_stride,
482 enum v3d_tiling_mode tiling_format, int cpp,
483 uint32_t image_h,
484 const struct pipe_box *box)
485 {
486 v3d_move_tiled_image(dst, dst_stride,
487 src, src_stride,
488 tiling_format,
489 cpp,
490 image_h,
491 box,
492 false);
493 }
494