1 /*
2 * Copyright 2007 VMware, Inc.
3 * Copyright 2016 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 /**
26 * \file
27 *
28 * Common helper functions for PBO up- and downloads.
29 */
30
31 #include "state_tracker/st_context.h"
32 #include "state_tracker/st_nir.h"
33 #include "state_tracker/st_pbo.h"
34
35 #include "main/context.h"
36 #include "pipe/p_context.h"
37 #include "pipe/p_defines.h"
38 #include "pipe/p_screen.h"
39 #include "cso_cache/cso_context.h"
40 #include "util/format/u_format.h"
41 #include "util/u_inlines.h"
42 #include "util/u_upload_mgr.h"
43
44 #include "compiler/nir/nir_builder.h"
45
46 /* Final setup of buffer addressing information.
47 *
48 * buf_offset is in pixels.
49 *
50 * Returns false if something (e.g. alignment) prevents PBO upload/download.
51 */
52 bool
st_pbo_addresses_setup(struct st_context * st,struct pipe_resource * buf,intptr_t buf_offset,struct st_pbo_addresses * addr)53 st_pbo_addresses_setup(struct st_context *st,
54 struct pipe_resource *buf, intptr_t buf_offset,
55 struct st_pbo_addresses *addr)
56 {
57 unsigned skip_pixels;
58
59 /* Check alignment against texture buffer requirements. */
60 {
61 unsigned ofs = (buf_offset * addr->bytes_per_pixel) % st->ctx->Const.TextureBufferOffsetAlignment;
62 if (ofs != 0) {
63 if (ofs % addr->bytes_per_pixel != 0)
64 return false;
65
66 skip_pixels = ofs / addr->bytes_per_pixel;
67 buf_offset -= skip_pixels;
68 } else {
69 skip_pixels = 0;
70 }
71 }
72
73 assert(buf_offset >= 0);
74
75 addr->buffer = buf;
76 addr->first_element = buf_offset;
77 addr->last_element = buf_offset + skip_pixels + addr->width - 1
78 + (addr->height - 1 + (addr->depth - 1) * addr->image_height) * addr->pixels_per_row;
79
80 if (addr->last_element - addr->first_element > st->ctx->Const.MaxTextureBufferSize - 1)
81 return false;
82
83 /* This should be ensured by Mesa before calling our callbacks */
84 assert((addr->last_element + 1) * addr->bytes_per_pixel <= buf->width0);
85
86 addr->constants.xoffset = -addr->xoffset + skip_pixels;
87 addr->constants.yoffset = -addr->yoffset;
88 addr->constants.stride = addr->pixels_per_row;
89 addr->constants.image_size = addr->pixels_per_row * addr->image_height;
90 addr->constants.layer_offset = 0;
91
92 return true;
93 }
94
95 /* Validate and fill buffer addressing information based on GL pixelstore
96 * attributes.
97 *
98 * Returns false if some aspect of the addressing (e.g. alignment) prevents
99 * PBO upload/download.
100 */
101 bool
st_pbo_addresses_pixelstore(struct st_context * st,GLenum gl_target,bool skip_images,const struct gl_pixelstore_attrib * store,const void * pixels,struct st_pbo_addresses * addr)102 st_pbo_addresses_pixelstore(struct st_context *st,
103 GLenum gl_target, bool skip_images,
104 const struct gl_pixelstore_attrib *store,
105 const void *pixels,
106 struct st_pbo_addresses *addr)
107 {
108 struct pipe_resource *buf = store->BufferObj->buffer;
109 intptr_t buf_offset = (intptr_t) pixels;
110
111 if (buf_offset % addr->bytes_per_pixel)
112 return false;
113
114 if (store->RowLength && store->RowLength < addr->width)
115 return false;
116
117 /* Convert to texels */
118 buf_offset = buf_offset / addr->bytes_per_pixel;
119
120 /* Determine image height */
121 if (gl_target == GL_TEXTURE_1D_ARRAY) {
122 addr->image_height = 1;
123 } else {
124 addr->image_height = store->ImageHeight > 0 ? store->ImageHeight : addr->height;
125 }
126
127 /* Compute the stride, taking store->Alignment into account */
128 {
129 unsigned pixels_per_row = store->RowLength > 0 ?
130 store->RowLength : addr->width;
131 unsigned bytes_per_row = pixels_per_row * addr->bytes_per_pixel;
132 unsigned remainder = bytes_per_row % store->Alignment;
133 unsigned offset_rows;
134
135 if (remainder > 0)
136 bytes_per_row += store->Alignment - remainder;
137
138 if (bytes_per_row % addr->bytes_per_pixel)
139 return false;
140
141 addr->pixels_per_row = bytes_per_row / addr->bytes_per_pixel;
142
143 offset_rows = store->SkipRows;
144 if (skip_images)
145 offset_rows += addr->image_height * store->SkipImages;
146
147 buf_offset += store->SkipPixels + addr->pixels_per_row * offset_rows;
148 }
149
150 if (!st_pbo_addresses_setup(st, buf, buf_offset, addr))
151 return false;
152
153 /* Support GL_PACK_INVERT_MESA */
154 if (store->Invert) {
155 addr->constants.xoffset += (addr->height - 1) * addr->constants.stride;
156 addr->constants.stride = -addr->constants.stride;
157 }
158
159 return true;
160 }
161
162 /* For download from a framebuffer, we may have to invert the Y axis. The
163 * setup is as follows:
164 * - set viewport to inverted, so that the position sysval is correct for
165 * texel fetches
166 * - this function adjusts the fragment shader's constant buffer to compute
167 * the correct destination addresses.
168 */
169 void
st_pbo_addresses_invert_y(struct st_pbo_addresses * addr,unsigned viewport_height)170 st_pbo_addresses_invert_y(struct st_pbo_addresses *addr,
171 unsigned viewport_height)
172 {
173 addr->constants.xoffset +=
174 (viewport_height - 1 + 2 * addr->constants.yoffset) * addr->constants.stride;
175 addr->constants.stride = -addr->constants.stride;
176 }
177
178 /* Setup all vertex pipeline state, rasterizer state, and fragment shader
179 * constants, and issue the draw call for PBO upload/download.
180 *
181 * The caller is responsible for saving and restoring state, as well as for
182 * setting other fragment shader state (fragment shader, samplers), and
183 * framebuffer/viewport/DSA/blend state.
184 */
185 bool
st_pbo_draw(struct st_context * st,const struct st_pbo_addresses * addr,unsigned surface_width,unsigned surface_height)186 st_pbo_draw(struct st_context *st, const struct st_pbo_addresses *addr,
187 unsigned surface_width, unsigned surface_height)
188 {
189 struct cso_context *cso = st->cso_context;
190 struct pipe_context *pipe = st->pipe;
191
192 /* Setup vertex and geometry shaders */
193 if (!st->pbo.vs) {
194 st->pbo.vs = st_pbo_create_vs(st);
195 if (!st->pbo.vs)
196 return false;
197 }
198
199 if (addr->depth != 1 && st->pbo.use_gs && !st->pbo.gs) {
200 st->pbo.gs = st_pbo_create_gs(st);
201 if (!st->pbo.gs)
202 return false;
203 }
204
205 cso_set_vertex_shader_handle(cso, st->pbo.vs);
206
207 cso_set_geometry_shader_handle(cso, addr->depth != 1 ? st->pbo.gs : NULL);
208
209 cso_set_tessctrl_shader_handle(cso, NULL);
210
211 cso_set_tesseval_shader_handle(cso, NULL);
212
213 /* Upload vertices */
214 {
215 struct pipe_vertex_buffer vbo = {0};
216 struct cso_velems_state velem;
217
218 float x0 = (float) addr->xoffset / surface_width * 2.0f - 1.0f;
219 float y0 = (float) addr->yoffset / surface_height * 2.0f - 1.0f;
220 float x1 = (float) (addr->xoffset + addr->width) / surface_width * 2.0f - 1.0f;
221 float y1 = (float) (addr->yoffset + addr->height) / surface_height * 2.0f - 1.0f;
222
223 float *verts = NULL;
224
225 u_upload_alloc(st->pipe->stream_uploader, 0, 8 * sizeof(float), 4,
226 &vbo.buffer_offset, &vbo.buffer.resource, (void **) &verts);
227 if (!verts)
228 return false;
229
230 verts[0] = x0;
231 verts[1] = y0;
232 verts[2] = x0;
233 verts[3] = y1;
234 verts[4] = x1;
235 verts[5] = y0;
236 verts[6] = x1;
237 verts[7] = y1;
238
239 u_upload_unmap(st->pipe->stream_uploader);
240
241 velem.count = 1;
242 velem.velems[0].src_offset = 0;
243 velem.velems[0].src_stride = 2 * sizeof(float);
244 velem.velems[0].instance_divisor = 0;
245 velem.velems[0].vertex_buffer_index = 0;
246 velem.velems[0].src_format = PIPE_FORMAT_R32G32_FLOAT;
247 velem.velems[0].dual_slot = false;
248
249 cso_set_vertex_elements(cso, &velem);
250 cso_set_vertex_buffers(cso, 1, true, &vbo);
251 }
252
253 /* Upload constants */
254 {
255 struct pipe_constant_buffer cb;
256
257 cb.buffer = NULL;
258 cb.user_buffer = &addr->constants;
259 cb.buffer_offset = 0;
260 cb.buffer_size = sizeof(addr->constants);
261
262 pipe->set_constant_buffer(pipe, PIPE_SHADER_FRAGMENT, 0, false, &cb);
263
264 pipe_resource_reference(&cb.buffer, NULL);
265 }
266
267 /* Rasterizer state */
268 cso_set_rasterizer(cso, &st->pbo.raster);
269
270 /* Disable stream output */
271 cso_set_stream_outputs(cso, 0, NULL, 0);
272
273 if (addr->depth == 1) {
274 cso_draw_arrays(cso, MESA_PRIM_TRIANGLE_STRIP, 0, 4);
275 } else {
276 cso_draw_arrays_instanced(cso, MESA_PRIM_TRIANGLE_STRIP,
277 0, 4, 0, addr->depth);
278 }
279
280 return true;
281 }
282
283 void *
st_pbo_create_vs(struct st_context * st)284 st_pbo_create_vs(struct st_context *st)
285 {
286 const struct glsl_type *vec4 = glsl_vec4_type();
287 const nir_shader_compiler_options *options =
288 st_get_nir_compiler_options(st, MESA_SHADER_VERTEX);
289
290 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX, options,
291 "st/pbo VS");
292
293 nir_variable *in_pos = nir_create_variable_with_location(b.shader, nir_var_shader_in,
294 VERT_ATTRIB_POS, vec4);
295
296 nir_variable *out_pos = nir_create_variable_with_location(b.shader, nir_var_shader_out,
297 VARYING_SLOT_POS, vec4);
298
299 if (!st->pbo.use_gs)
300 nir_copy_var(&b, out_pos, in_pos);
301
302 if (st->pbo.layers) {
303 nir_variable *instance_id = nir_create_variable_with_location(b.shader, nir_var_system_value,
304 SYSTEM_VALUE_INSTANCE_ID, glsl_int_type());
305
306 if (st->pbo.use_gs) {
307 nir_store_var(&b, out_pos,
308 nir_vector_insert_imm(&b, nir_load_var(&b, in_pos),
309 nir_i2f32(&b, nir_load_var(&b, instance_id)), 2),
310 0xf);
311 } else {
312 nir_variable *out_layer = nir_create_variable_with_location(b.shader, nir_var_shader_out,
313 VARYING_SLOT_LAYER, glsl_int_type());
314 out_layer->data.interpolation = INTERP_MODE_NONE;
315 nir_copy_var(&b, out_layer, instance_id);
316 }
317 }
318
319 return st_nir_finish_builtin_shader(st, b.shader);
320 }
321
322 void *
st_pbo_create_gs(struct st_context * st)323 st_pbo_create_gs(struct st_context *st)
324 {
325 const nir_shader_compiler_options *options =
326 st_get_nir_compiler_options(st, MESA_SHADER_GEOMETRY);
327
328 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_GEOMETRY, options,
329 "st/pbo GS");
330
331 b.shader->info.gs.input_primitive = MESA_PRIM_TRIANGLES;
332 b.shader->info.gs.output_primitive = MESA_PRIM_TRIANGLE_STRIP;
333 b.shader->info.gs.vertices_in = 3;
334 b.shader->info.gs.vertices_out = 3;
335 b.shader->info.gs.invocations = 1;
336 b.shader->info.gs.active_stream_mask = 1;
337
338 const struct glsl_type *in_type = glsl_array_type(glsl_vec4_type(), 3, 0);
339 nir_variable *in_pos = nir_variable_create(b.shader, nir_var_shader_in, in_type, "in_pos");
340 in_pos->data.location = VARYING_SLOT_POS;
341 b.shader->info.inputs_read |= VARYING_BIT_POS;
342
343 nir_variable *out_pos = nir_create_variable_with_location(b.shader, nir_var_shader_out,
344 VARYING_SLOT_POS, glsl_vec4_type());
345
346 b.shader->info.outputs_written |= VARYING_BIT_POS;
347
348 nir_variable *out_layer = nir_create_variable_with_location(b.shader, nir_var_shader_out,
349 VARYING_SLOT_LAYER, glsl_int_type());
350 out_layer->data.interpolation = INTERP_MODE_NONE;
351 b.shader->info.outputs_written |= VARYING_BIT_LAYER;
352
353 for (int i = 0; i < 3; ++i) {
354 nir_def *pos = nir_load_array_var_imm(&b, in_pos, i);
355
356 nir_store_var(&b, out_pos, nir_vector_insert_imm(&b, pos, nir_imm_float(&b, 0.0), 2), 0xf);
357 /* out_layer.x = f2i(in_pos[i].z) */
358 nir_store_var(&b, out_layer, nir_f2i32(&b, nir_channel(&b, pos, 2)), 0x1);
359
360 nir_emit_vertex(&b);
361 }
362
363 return st_nir_finish_builtin_shader(st, b.shader);
364 }
365
366 const struct glsl_type *
st_pbo_sampler_type_for_target(enum pipe_texture_target target,enum st_pbo_conversion conv)367 st_pbo_sampler_type_for_target(enum pipe_texture_target target,
368 enum st_pbo_conversion conv)
369 {
370 bool is_array = target >= PIPE_TEXTURE_1D_ARRAY;
371 static const enum glsl_sampler_dim dim[] = {
372 [PIPE_BUFFER] = GLSL_SAMPLER_DIM_BUF,
373 [PIPE_TEXTURE_1D] = GLSL_SAMPLER_DIM_1D,
374 [PIPE_TEXTURE_2D] = GLSL_SAMPLER_DIM_2D,
375 [PIPE_TEXTURE_3D] = GLSL_SAMPLER_DIM_3D,
376 [PIPE_TEXTURE_CUBE] = GLSL_SAMPLER_DIM_CUBE,
377 [PIPE_TEXTURE_RECT] = GLSL_SAMPLER_DIM_RECT,
378 [PIPE_TEXTURE_1D_ARRAY] = GLSL_SAMPLER_DIM_1D,
379 [PIPE_TEXTURE_2D_ARRAY] = GLSL_SAMPLER_DIM_2D,
380 [PIPE_TEXTURE_CUBE_ARRAY] = GLSL_SAMPLER_DIM_CUBE,
381 };
382
383 static const enum glsl_base_type type[] = {
384 [ST_PBO_CONVERT_FLOAT] = GLSL_TYPE_FLOAT,
385 [ST_PBO_CONVERT_UINT] = GLSL_TYPE_UINT,
386 [ST_PBO_CONVERT_UINT_TO_SINT] = GLSL_TYPE_UINT,
387 [ST_PBO_CONVERT_SINT] = GLSL_TYPE_INT,
388 [ST_PBO_CONVERT_SINT_TO_UINT] = GLSL_TYPE_INT,
389 };
390
391 return glsl_sampler_type(dim[target], false, is_array, type[conv]);
392 }
393
394
395 static void *
create_fs(struct st_context * st,bool download,enum pipe_texture_target target,enum st_pbo_conversion conversion,enum pipe_format format,bool need_layer)396 create_fs(struct st_context *st, bool download,
397 enum pipe_texture_target target,
398 enum st_pbo_conversion conversion,
399 enum pipe_format format,
400 bool need_layer)
401 {
402 struct pipe_screen *screen = st->screen;
403 const nir_shader_compiler_options *options =
404 st_get_nir_compiler_options(st, MESA_SHADER_FRAGMENT);
405 bool pos_is_sysval =
406 screen->get_param(screen, PIPE_CAP_FS_POSITION_IS_SYSVAL);
407
408 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, options,
409 download ?
410 "st/pbo download FS" :
411 "st/pbo upload FS");
412
413 nir_def *zero = nir_imm_int(&b, 0);
414
415 /* param = [ -xoffset + skip_pixels, -yoffset, stride, image_height ] */
416 nir_variable *param_var =
417 nir_variable_create(b.shader, nir_var_uniform, glsl_vec4_type(), "param");
418 b.shader->num_uniforms += 4;
419 nir_def *param = nir_load_var(&b, param_var);
420
421 nir_variable *fragcoord;
422 if (pos_is_sysval)
423 fragcoord = nir_create_variable_with_location(b.shader, nir_var_system_value,
424 SYSTEM_VALUE_FRAG_COORD, glsl_vec4_type());
425 else
426 fragcoord = nir_create_variable_with_location(b.shader, nir_var_shader_in,
427 VARYING_SLOT_POS, glsl_vec4_type());
428 nir_def *coord = nir_load_var(&b, fragcoord);
429
430 /* When st->pbo.layers == false, it is guaranteed we only have a single
431 * layer. But we still need the "layer" variable to add the "array"
432 * coordinate to the texture. Hence we set layer to zero when array texture
433 * is used in case only a single layer is required.
434 */
435 nir_def *layer = NULL;
436 if (!download || target == PIPE_TEXTURE_1D_ARRAY ||
437 target == PIPE_TEXTURE_2D_ARRAY ||
438 target == PIPE_TEXTURE_3D ||
439 target == PIPE_TEXTURE_CUBE ||
440 target == PIPE_TEXTURE_CUBE_ARRAY) {
441 if (need_layer) {
442 assert(st->pbo.layers);
443 nir_variable *var = nir_create_variable_with_location(b.shader, nir_var_shader_in,
444 VARYING_SLOT_LAYER, glsl_int_type());
445 var->data.interpolation = INTERP_MODE_FLAT;
446 layer = nir_load_var(&b, var);
447 }
448 else {
449 layer = zero;
450 }
451 }
452
453 /* offset_pos = param.xy + f2i(coord.xy) */
454 nir_def *offset_pos =
455 nir_iadd(&b, nir_channels(&b, param, TGSI_WRITEMASK_XY),
456 nir_f2i32(&b, nir_channels(&b, coord, TGSI_WRITEMASK_XY)));
457
458 /* addr = offset_pos.x + offset_pos.y * stride */
459 nir_def *pbo_addr =
460 nir_iadd(&b, nir_channel(&b, offset_pos, 0),
461 nir_imul(&b, nir_channel(&b, offset_pos, 1),
462 nir_channel(&b, param, 2)));
463 if (layer && layer != zero) {
464 /* pbo_addr += image_height * layer */
465 pbo_addr = nir_iadd(&b, pbo_addr,
466 nir_imul(&b, layer, nir_channel(&b, param, 3)));
467 }
468
469 nir_def *texcoord;
470 if (download) {
471 texcoord = nir_f2i32(&b, nir_channels(&b, coord, TGSI_WRITEMASK_XY));
472
473 if (target == PIPE_TEXTURE_1D) {
474 unsigned sw = 0;
475 texcoord = nir_swizzle(&b, texcoord, &sw, 1);
476 }
477
478 if (layer) {
479 nir_def *src_layer = layer;
480
481 if (target == PIPE_TEXTURE_3D) {
482 nir_variable *layer_offset_var =
483 nir_variable_create(b.shader, nir_var_uniform,
484 glsl_int_type(), "layer_offset");
485 b.shader->num_uniforms += 1;
486 layer_offset_var->data.driver_location = 4;
487 nir_def *layer_offset = nir_load_var(&b, layer_offset_var);
488
489 src_layer = nir_iadd(&b, layer, layer_offset);
490 }
491
492 if (target == PIPE_TEXTURE_1D_ARRAY) {
493 texcoord = nir_vec2(&b, nir_channel(&b, texcoord, 0),
494 src_layer);
495 } else {
496 texcoord = nir_vec3(&b, nir_channel(&b, texcoord, 0),
497 nir_channel(&b, texcoord, 1),
498 src_layer);
499 }
500 }
501 } else {
502 texcoord = pbo_addr;
503 }
504
505 nir_variable *tex_var =
506 nir_variable_create(b.shader, nir_var_uniform,
507 st_pbo_sampler_type_for_target(target, conversion),
508 "tex");
509 tex_var->data.explicit_binding = true;
510 tex_var->data.binding = 0;
511
512 nir_deref_instr *tex_deref = nir_build_deref_var(&b, tex_var);
513
514 nir_tex_instr *tex = nir_tex_instr_create(b.shader, 3);
515 tex->op = nir_texop_txf;
516 tex->sampler_dim = glsl_get_sampler_dim(tex_var->type);
517 tex->coord_components =
518 glsl_get_sampler_coordinate_components(tex_var->type);
519 tex->is_array = target >= PIPE_TEXTURE_1D_ARRAY;
520
521 tex->dest_type = nir_get_nir_type_for_glsl_base_type(glsl_get_sampler_result_type(tex_var->type));
522 tex->src[0].src_type = nir_tex_src_texture_deref;
523 tex->src[0].src = nir_src_for_ssa(&tex_deref->def);
524 tex->src[1].src_type = nir_tex_src_sampler_deref;
525 tex->src[1].src = nir_src_for_ssa(&tex_deref->def);
526 tex->src[2].src_type = nir_tex_src_coord;
527 tex->src[2].src = nir_src_for_ssa(texcoord);
528 nir_def_init(&tex->instr, &tex->def, 4, 32);
529 nir_builder_instr_insert(&b, &tex->instr);
530 nir_def *result = &tex->def;
531
532 if (conversion == ST_PBO_CONVERT_SINT_TO_UINT)
533 result = nir_imax(&b, result, zero);
534 else if (conversion == ST_PBO_CONVERT_UINT_TO_SINT)
535 result = nir_umin(&b, result, nir_imm_int(&b, (1u << 31) - 1));
536
537 if (download) {
538 static const enum glsl_base_type type[] = {
539 [ST_PBO_CONVERT_FLOAT] = GLSL_TYPE_FLOAT,
540 [ST_PBO_CONVERT_UINT] = GLSL_TYPE_UINT,
541 [ST_PBO_CONVERT_UINT_TO_SINT] = GLSL_TYPE_INT,
542 [ST_PBO_CONVERT_SINT] = GLSL_TYPE_INT,
543 [ST_PBO_CONVERT_SINT_TO_UINT] = GLSL_TYPE_UINT,
544 };
545 static const nir_alu_type nir_types[] = {
546 [ST_PBO_CONVERT_FLOAT] = nir_type_float,
547 [ST_PBO_CONVERT_UINT] = nir_type_uint,
548 [ST_PBO_CONVERT_UINT_TO_SINT] = nir_type_int,
549 [ST_PBO_CONVERT_SINT] = nir_type_int,
550 [ST_PBO_CONVERT_SINT_TO_UINT] = nir_type_uint,
551 };
552 nir_variable *img_var =
553 nir_variable_create(b.shader, nir_var_image,
554 glsl_image_type(GLSL_SAMPLER_DIM_BUF, false,
555 type[conversion]), "img");
556 img_var->data.access = ACCESS_NON_READABLE;
557 img_var->data.explicit_binding = true;
558 img_var->data.binding = 0;
559 img_var->data.image.format = format;
560 nir_deref_instr *img_deref = nir_build_deref_var(&b, img_var);
561
562 nir_image_deref_store(&b, &img_deref->def,
563 nir_vec4(&b, pbo_addr, zero, zero, zero),
564 zero,
565 result,
566 nir_imm_int(&b, 0),
567 .src_type = nir_types[conversion],
568 .image_dim = GLSL_SAMPLER_DIM_BUF);
569 } else {
570 nir_variable *color =
571 nir_create_variable_with_location(b.shader, nir_var_shader_out,
572 FRAG_RESULT_COLOR, glsl_vec4_type());
573
574 nir_store_var(&b, color, result, TGSI_WRITEMASK_XYZW);
575 }
576
577 return st_nir_finish_builtin_shader(st, b.shader);
578 }
579
580 static enum st_pbo_conversion
get_pbo_conversion(enum pipe_format src_format,enum pipe_format dst_format)581 get_pbo_conversion(enum pipe_format src_format, enum pipe_format dst_format)
582 {
583 if (util_format_is_pure_uint(src_format)) {
584 if (util_format_is_pure_uint(dst_format))
585 return ST_PBO_CONVERT_UINT;
586 if (util_format_is_pure_sint(dst_format))
587 return ST_PBO_CONVERT_UINT_TO_SINT;
588 } else if (util_format_is_pure_sint(src_format)) {
589 if (util_format_is_pure_sint(dst_format))
590 return ST_PBO_CONVERT_SINT;
591 if (util_format_is_pure_uint(dst_format))
592 return ST_PBO_CONVERT_SINT_TO_UINT;
593 }
594
595 return ST_PBO_CONVERT_FLOAT;
596 }
597
598 void *
st_pbo_get_upload_fs(struct st_context * st,enum pipe_format src_format,enum pipe_format dst_format,bool need_layer)599 st_pbo_get_upload_fs(struct st_context *st,
600 enum pipe_format src_format,
601 enum pipe_format dst_format,
602 bool need_layer)
603 {
604 STATIC_ASSERT(ARRAY_SIZE(st->pbo.upload_fs) == ST_NUM_PBO_CONVERSIONS);
605
606 enum st_pbo_conversion conversion = get_pbo_conversion(src_format, dst_format);
607
608 if (!st->pbo.upload_fs[conversion][need_layer])
609 st->pbo.upload_fs[conversion][need_layer] = create_fs(st, false, 0, conversion, PIPE_FORMAT_NONE, need_layer);
610
611 return st->pbo.upload_fs[conversion][need_layer];
612 }
613
614 void *
st_pbo_get_download_fs(struct st_context * st,enum pipe_texture_target target,enum pipe_format src_format,enum pipe_format dst_format,bool need_layer)615 st_pbo_get_download_fs(struct st_context *st, enum pipe_texture_target target,
616 enum pipe_format src_format,
617 enum pipe_format dst_format,
618 bool need_layer)
619 {
620 STATIC_ASSERT(ARRAY_SIZE(st->pbo.download_fs) == ST_NUM_PBO_CONVERSIONS);
621 assert(target < PIPE_MAX_TEXTURE_TYPES);
622
623 struct pipe_screen *screen = st->screen;
624 enum st_pbo_conversion conversion = get_pbo_conversion(src_format, dst_format);
625 bool formatless_store = screen->get_param(screen, PIPE_CAP_IMAGE_STORE_FORMATTED);
626
627 /* For drivers not supporting formatless storing, download FS is stored in an
628 * indirect dynamically allocated array of storing formats.
629 */
630 if (!formatless_store && !st->pbo.download_fs[conversion][target][need_layer])
631 st->pbo.download_fs[conversion][target][need_layer] = calloc(sizeof(void *), PIPE_FORMAT_COUNT);
632
633 if (formatless_store) {
634 if (!st->pbo.download_fs[conversion][target][need_layer])
635 st->pbo.download_fs[conversion][target][need_layer] = create_fs(st, true, target, conversion, PIPE_FORMAT_NONE, need_layer);
636 return st->pbo.download_fs[conversion][target][need_layer];
637 } else {
638 void **fs_array = (void **)st->pbo.download_fs[conversion][target][need_layer];
639 if (!fs_array[dst_format])
640 fs_array[dst_format] = create_fs(st, true, target, conversion, dst_format, need_layer);
641 return fs_array[dst_format];
642 }
643 }
644
645 void
st_init_pbo_helpers(struct st_context * st)646 st_init_pbo_helpers(struct st_context *st)
647 {
648 struct pipe_screen *screen = st->screen;
649
650 st->pbo.upload_enabled =
651 screen->get_param(screen, PIPE_CAP_TEXTURE_BUFFER_OBJECTS) &&
652 screen->get_param(screen, PIPE_CAP_TEXTURE_BUFFER_OFFSET_ALIGNMENT) >= 1 &&
653 screen->get_shader_param(screen, PIPE_SHADER_FRAGMENT, PIPE_SHADER_CAP_INTEGERS);
654 if (!st->pbo.upload_enabled)
655 return;
656
657 st->pbo.download_enabled =
658 st->pbo.upload_enabled &&
659 screen->get_param(screen, PIPE_CAP_SAMPLER_VIEW_TARGET) &&
660 screen->get_param(screen, PIPE_CAP_FRAMEBUFFER_NO_ATTACHMENT) &&
661 screen->get_shader_param(screen, PIPE_SHADER_FRAGMENT,
662 PIPE_SHADER_CAP_MAX_SHADER_IMAGES) >= 1;
663
664 st->pbo.rgba_only =
665 screen->get_param(screen, PIPE_CAP_BUFFER_SAMPLER_VIEW_RGBA_ONLY);
666
667 if (screen->get_param(screen, PIPE_CAP_VS_INSTANCEID)) {
668 if (screen->get_param(screen, PIPE_CAP_VS_LAYER_VIEWPORT)) {
669 st->pbo.layers = true;
670 } else if (screen->get_param(screen, PIPE_CAP_MAX_GEOMETRY_OUTPUT_VERTICES) >= 3) {
671 st->pbo.layers = true;
672 st->pbo.use_gs = true;
673 }
674 }
675
676 /* Blend state */
677 memset(&st->pbo.upload_blend, 0, sizeof(struct pipe_blend_state));
678 st->pbo.upload_blend.rt[0].colormask = PIPE_MASK_RGBA;
679
680 /* Rasterizer state */
681 memset(&st->pbo.raster, 0, sizeof(struct pipe_rasterizer_state));
682 st->pbo.raster.half_pixel_center = 1;
683
684 const char *pbo = debug_get_option("MESA_COMPUTE_PBO", NULL);
685 if (pbo) {
686 st->force_compute_based_texture_transfer = true;
687 st->force_specialized_compute_transfer = !strncmp(pbo, "spec", 4);
688 }
689
690 if (st->allow_compute_based_texture_transfer || st->force_compute_based_texture_transfer)
691 st->pbo.shaders = _mesa_hash_table_create_u32_keys(NULL);
692 }
693
694 void
st_destroy_pbo_helpers(struct st_context * st)695 st_destroy_pbo_helpers(struct st_context *st)
696 {
697 struct pipe_screen *screen = st->screen;
698 bool formatless_store = screen->get_param(screen, PIPE_CAP_IMAGE_STORE_FORMATTED);
699 unsigned i;
700
701 for (i = 0; i < ARRAY_SIZE(st->pbo.upload_fs); ++i) {
702 for (unsigned j = 0; j < ARRAY_SIZE(st->pbo.upload_fs[0]); j++) {
703 if (st->pbo.upload_fs[i][j]) {
704 st->pipe->delete_fs_state(st->pipe, st->pbo.upload_fs[i][j]);
705 st->pbo.upload_fs[i][j] = NULL;
706 }
707 }
708 }
709
710 for (i = 0; i < ARRAY_SIZE(st->pbo.download_fs); ++i) {
711 for (unsigned j = 0; j < ARRAY_SIZE(st->pbo.download_fs[0]); ++j) {
712 for (unsigned k = 0; k < ARRAY_SIZE(st->pbo.download_fs[0][0]); k++) {
713 if (st->pbo.download_fs[i][j][k]) {
714 if (formatless_store) {
715 st->pipe->delete_fs_state(st->pipe, st->pbo.download_fs[i][j][k]);
716 } else {
717 void **fs_array = (void **)st->pbo.download_fs[i][j][k];
718 for (unsigned l = 0; l < PIPE_FORMAT_COUNT; l++)
719 if (fs_array[l])
720 st->pipe->delete_fs_state(st->pipe, fs_array[l]);
721 free(st->pbo.download_fs[i][j][k]);
722 }
723 st->pbo.download_fs[i][j][k] = NULL;
724 }
725 }
726 }
727 }
728
729 if (st->pbo.gs) {
730 st->pipe->delete_gs_state(st->pipe, st->pbo.gs);
731 st->pbo.gs = NULL;
732 }
733
734 if (st->pbo.vs) {
735 st->pipe->delete_vs_state(st->pipe, st->pbo.vs);
736 st->pbo.vs = NULL;
737 }
738
739 st_pbo_compute_deinit(st);
740 }
741