1 /**************************************************************************
2 *
3 * Copyright 2011 Marek Olšák <[email protected]>
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /**
29 * This module uploads user buffers and translates the vertex buffers which
30 * contain incompatible vertices (i.e. not supported by the driver/hardware)
31 * into compatible ones, based on the Gallium CAPs.
32 *
33 * It does not upload index buffers.
34 *
35 * The module heavily uses bitmasks to represent per-buffer and
36 * per-vertex-element flags to avoid looping over the list of buffers just
37 * to see if there's a non-zero stride, or user buffer, or unsupported format,
38 * etc.
39 *
40 * There are 3 categories of vertex elements, which are processed separately:
41 * - per-vertex attribs (stride != 0, instance_divisor == 0)
42 * - instanced attribs (stride != 0, instance_divisor > 0)
43 * - constant attribs (stride == 0)
44 *
45 * All needed uploads and translations are performed every draw command, but
46 * only the subset of vertices needed for that draw command is uploaded or
47 * translated. (the module never translates whole buffers)
48 *
49 *
50 * The module consists of two main parts:
51 *
52 *
53 * 1) Translate (u_vbuf_translate_begin/end)
54 *
55 * This is pretty much a vertex fetch fallback. It translates vertices from
56 * one vertex buffer to another in an unused vertex buffer slot. It does
57 * whatever is needed to make the vertices readable by the hardware (changes
58 * vertex formats and aligns offsets and strides). The translate module is
59 * used here.
60 *
61 * Each of the 3 categories is translated to a separate buffer.
62 * Only the [min_index, max_index] range is translated. For instanced attribs,
63 * the range is [start_instance, start_instance+instance_count]. For constant
64 * attribs, the range is [0, 1].
65 *
66 *
67 * 2) User buffer uploading (u_vbuf_upload_buffers)
68 *
69 * Only the [min_index, max_index] range is uploaded (just like Translate)
70 * with a single memcpy.
71 *
72 * This method works best for non-indexed draw operations or indexed draw
73 * operations where the [min_index, max_index] range is not being way bigger
74 * than the vertex count.
75 *
76 * If the range is too big (e.g. one triangle with indices {0, 1, 10000}),
77 * the per-vertex attribs are uploaded via the translate module, all packed
78 * into one vertex buffer, and the indexed draw call is turned into
79 * a non-indexed one in the process. This adds additional complexity
80 * to the translate part, but it prevents bad apps from bringing your frame
81 * rate down.
82 *
83 *
84 * If there is nothing to do, it forwards every command to the driver.
85 * The module also has its own CSO cache of vertex element states.
86 */
87
88 #include "util/u_vbuf.h"
89
90 #include "util/u_dump.h"
91 #include "util/format/u_format.h"
92 #include "util/u_helpers.h"
93 #include "util/u_inlines.h"
94 #include "util/u_memory.h"
95 #include "util/u_prim_restart.h"
96 #include "util/u_screen.h"
97 #include "util/u_upload_mgr.h"
98 #include "indices/u_primconvert.h"
99 #include "translate/translate.h"
100 #include "translate/translate_cache.h"
101 #include "cso_cache/cso_cache.h"
102 #include "cso_cache/cso_hash.h"
103
104 struct u_vbuf_elements {
105 unsigned count;
106 struct pipe_vertex_element ve[PIPE_MAX_ATTRIBS];
107
108 unsigned src_format_size[PIPE_MAX_ATTRIBS];
109
110 /* If (velem[i].src_format != native_format[i]), the vertex buffer
111 * referenced by the vertex element cannot be used for rendering and
112 * its vertex data must be translated to native_format[i]. */
113 enum pipe_format native_format[PIPE_MAX_ATTRIBS];
114 unsigned native_format_size[PIPE_MAX_ATTRIBS];
115 unsigned component_size[PIPE_MAX_ATTRIBS];
116 /* buffer-indexed */
117 unsigned strides[PIPE_MAX_ATTRIBS];
118
119 /* Which buffers are used by the vertex element state. */
120 uint32_t used_vb_mask;
121 /* This might mean two things:
122 * - src_format != native_format, as discussed above.
123 * - src_offset % 4 != 0 (if the caps don't allow such an offset). */
124 uint32_t incompatible_elem_mask; /* each bit describes a corresp. attrib */
125 /* Which buffer has at least one vertex element referencing it
126 * incompatible. */
127 uint32_t incompatible_vb_mask_any;
128 /* Which buffer has all vertex elements referencing it incompatible. */
129 uint32_t incompatible_vb_mask_all;
130 /* Which buffer has at least one vertex element referencing it
131 * compatible. */
132 uint32_t compatible_vb_mask_any;
133 uint32_t vb_align_mask[2]; //which buffers require 2/4 byte alignments
134 /* Which buffer has all vertex elements referencing it compatible. */
135 uint32_t compatible_vb_mask_all;
136
137 /* Which buffer has at least one vertex element referencing it
138 * non-instanced. */
139 uint32_t noninstance_vb_mask_any;
140
141 /* Which buffers are used by multiple vertex attribs. */
142 uint32_t interleaved_vb_mask;
143
144 /* Which buffer has a non-zero stride. */
145 uint32_t nonzero_stride_vb_mask; /* each bit describes a corresp. buffer */
146
147 /* Which buffer is incompatible (unaligned). */
148 uint32_t incompatible_vb_mask; /* each bit describes a corresp. buffer */
149
150 void *driver_cso;
151 };
152
153 enum {
154 VB_VERTEX = 0,
155 VB_INSTANCE = 1,
156 VB_CONST = 2,
157 VB_NUM = 3
158 };
159
160 struct u_vbuf {
161 struct u_vbuf_caps caps;
162 bool has_signed_vb_offset;
163
164 struct pipe_context *pipe;
165 struct translate_cache *translate_cache;
166 struct cso_cache cso_cache;
167
168 struct primconvert_context *pc;
169 bool flatshade_first;
170
171 /* This is what was set in set_vertex_buffers.
172 * May contain user buffers. */
173 struct pipe_vertex_buffer vertex_buffer[PIPE_MAX_ATTRIBS];
174 uint8_t num_vertex_buffers;
175 uint8_t num_real_vertex_buffers;
176 bool vertex_buffers_dirty;
177 uint32_t enabled_vb_mask;
178
179 uint32_t unaligned_vb_mask[2]; //16/32bit
180
181 /* Vertex buffers for the driver.
182 * There are usually no user buffers. */
183 struct pipe_vertex_buffer real_vertex_buffer[PIPE_MAX_ATTRIBS];
184
185 /* Vertex elements. */
186 struct u_vbuf_elements *ve, *ve_saved;
187
188 /* Vertex elements used for the translate fallback. */
189 struct cso_velems_state fallback_velems;
190 /* If non-NULL, this is a vertex element state used for the translate
191 * fallback and therefore used for rendering too. */
192 bool using_translate;
193 /* The vertex buffer slot index where translated vertices have been
194 * stored in. */
195 unsigned fallback_vbs[VB_NUM];
196 unsigned fallback_vbs_mask;
197
198 /* Which buffer is a user buffer. */
199 uint32_t user_vb_mask; /* each bit describes a corresp. buffer */
200 /* Which buffer is incompatible (unaligned). */
201 uint32_t incompatible_vb_mask; /* each bit describes a corresp. buffer */
202 /* Which buffers are allowed (supported by hardware). */
203 uint32_t allowed_vb_mask;
204 };
205
206 static void *
207 u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
208 const struct pipe_vertex_element *attribs);
209 static void u_vbuf_delete_vertex_elements(void *ctx, void *state,
210 enum cso_cache_type type);
211
212 static const struct {
213 enum pipe_format from, to;
214 } vbuf_format_fallbacks[] = {
215 { PIPE_FORMAT_R32_FIXED, PIPE_FORMAT_R32_FLOAT },
216 { PIPE_FORMAT_R32G32_FIXED, PIPE_FORMAT_R32G32_FLOAT },
217 { PIPE_FORMAT_R32G32B32_FIXED, PIPE_FORMAT_R32G32B32_FLOAT },
218 { PIPE_FORMAT_R32G32B32A32_FIXED, PIPE_FORMAT_R32G32B32A32_FLOAT },
219 { PIPE_FORMAT_R16_FLOAT, PIPE_FORMAT_R32_FLOAT },
220 { PIPE_FORMAT_R16G16_FLOAT, PIPE_FORMAT_R32G32_FLOAT },
221 { PIPE_FORMAT_R16G16B16_FLOAT, PIPE_FORMAT_R32G32B32_FLOAT },
222 { PIPE_FORMAT_R16G16B16A16_FLOAT, PIPE_FORMAT_R32G32B32A32_FLOAT },
223 { PIPE_FORMAT_R64_FLOAT, PIPE_FORMAT_R32_FLOAT },
224 { PIPE_FORMAT_R64G64_FLOAT, PIPE_FORMAT_R32G32_FLOAT },
225 { PIPE_FORMAT_R64G64B64_FLOAT, PIPE_FORMAT_R32G32B32_FLOAT },
226 { PIPE_FORMAT_R64G64B64A64_FLOAT, PIPE_FORMAT_R32G32B32A32_FLOAT },
227 { PIPE_FORMAT_R32_UNORM, PIPE_FORMAT_R32_FLOAT },
228 { PIPE_FORMAT_R32G32_UNORM, PIPE_FORMAT_R32G32_FLOAT },
229 { PIPE_FORMAT_R32G32B32_UNORM, PIPE_FORMAT_R32G32B32_FLOAT },
230 { PIPE_FORMAT_R32G32B32A32_UNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
231 { PIPE_FORMAT_R32_SNORM, PIPE_FORMAT_R32_FLOAT },
232 { PIPE_FORMAT_R32G32_SNORM, PIPE_FORMAT_R32G32_FLOAT },
233 { PIPE_FORMAT_R32G32B32_SNORM, PIPE_FORMAT_R32G32B32_FLOAT },
234 { PIPE_FORMAT_R32G32B32A32_SNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
235 { PIPE_FORMAT_R32_USCALED, PIPE_FORMAT_R32_FLOAT },
236 { PIPE_FORMAT_R32G32_USCALED, PIPE_FORMAT_R32G32_FLOAT },
237 { PIPE_FORMAT_R32G32B32_USCALED, PIPE_FORMAT_R32G32B32_FLOAT },
238 { PIPE_FORMAT_R32G32B32A32_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
239 { PIPE_FORMAT_R32_SSCALED, PIPE_FORMAT_R32_FLOAT },
240 { PIPE_FORMAT_R32G32_SSCALED, PIPE_FORMAT_R32G32_FLOAT },
241 { PIPE_FORMAT_R32G32B32_SSCALED, PIPE_FORMAT_R32G32B32_FLOAT },
242 { PIPE_FORMAT_R32G32B32A32_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
243 { PIPE_FORMAT_R16_UNORM, PIPE_FORMAT_R32_FLOAT },
244 { PIPE_FORMAT_R16G16_UNORM, PIPE_FORMAT_R32G32_FLOAT },
245 { PIPE_FORMAT_R16G16B16_UNORM, PIPE_FORMAT_R32G32B32_FLOAT },
246 { PIPE_FORMAT_R16G16B16A16_UNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
247 { PIPE_FORMAT_R16_SNORM, PIPE_FORMAT_R32_FLOAT },
248 { PIPE_FORMAT_R16G16_SNORM, PIPE_FORMAT_R32G32_FLOAT },
249 { PIPE_FORMAT_R16G16B16_SNORM, PIPE_FORMAT_R32G32B32_FLOAT },
250 { PIPE_FORMAT_R16G16B16_SINT, PIPE_FORMAT_R32G32B32_SINT },
251 { PIPE_FORMAT_R16G16B16_UINT, PIPE_FORMAT_R32G32B32_UINT },
252 { PIPE_FORMAT_R16G16B16A16_SNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
253 { PIPE_FORMAT_R16_USCALED, PIPE_FORMAT_R32_FLOAT },
254 { PIPE_FORMAT_R16G16_USCALED, PIPE_FORMAT_R32G32_FLOAT },
255 { PIPE_FORMAT_R16G16B16_USCALED, PIPE_FORMAT_R32G32B32_FLOAT },
256 { PIPE_FORMAT_R16G16B16A16_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
257 { PIPE_FORMAT_R16_SSCALED, PIPE_FORMAT_R32_FLOAT },
258 { PIPE_FORMAT_R16G16_SSCALED, PIPE_FORMAT_R32G32_FLOAT },
259 { PIPE_FORMAT_R16G16B16_SSCALED, PIPE_FORMAT_R32G32B32_FLOAT },
260 { PIPE_FORMAT_R16G16B16A16_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
261 { PIPE_FORMAT_R8_UNORM, PIPE_FORMAT_R32_FLOAT },
262 { PIPE_FORMAT_R8G8_UNORM, PIPE_FORMAT_R32G32_FLOAT },
263 { PIPE_FORMAT_R8G8B8_UNORM, PIPE_FORMAT_R32G32B32_FLOAT },
264 { PIPE_FORMAT_R8G8B8A8_UNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
265 { PIPE_FORMAT_R8_SNORM, PIPE_FORMAT_R32_FLOAT },
266 { PIPE_FORMAT_R8G8_SNORM, PIPE_FORMAT_R32G32_FLOAT },
267 { PIPE_FORMAT_R8G8B8_SNORM, PIPE_FORMAT_R32G32B32_FLOAT },
268 { PIPE_FORMAT_R8G8B8A8_SNORM, PIPE_FORMAT_R32G32B32A32_FLOAT },
269 { PIPE_FORMAT_R8_USCALED, PIPE_FORMAT_R32_FLOAT },
270 { PIPE_FORMAT_R8G8_USCALED, PIPE_FORMAT_R32G32_FLOAT },
271 { PIPE_FORMAT_R8G8B8_USCALED, PIPE_FORMAT_R32G32B32_FLOAT },
272 { PIPE_FORMAT_R8G8B8A8_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
273 { PIPE_FORMAT_R8_SSCALED, PIPE_FORMAT_R32_FLOAT },
274 { PIPE_FORMAT_R8G8_SSCALED, PIPE_FORMAT_R32G32_FLOAT },
275 { PIPE_FORMAT_R8G8B8_SSCALED, PIPE_FORMAT_R32G32B32_FLOAT },
276 { PIPE_FORMAT_R8G8B8A8_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
277 };
278
u_vbuf_get_caps(struct pipe_screen * screen,struct u_vbuf_caps * caps,bool needs64b)279 void u_vbuf_get_caps(struct pipe_screen *screen, struct u_vbuf_caps *caps,
280 bool needs64b)
281 {
282 unsigned i;
283
284 memset(caps, 0, sizeof(*caps));
285
286 /* I'd rather have a bitfield of which formats are supported and a static
287 * table of the translations indexed by format, but since we don't have C99
288 * we can't easily make a sparsely-populated table indexed by format. So,
289 * we construct the sparse table here.
290 */
291 for (i = 0; i < PIPE_FORMAT_COUNT; i++)
292 caps->format_translation[i] = i;
293
294 for (i = 0; i < ARRAY_SIZE(vbuf_format_fallbacks); i++) {
295 enum pipe_format format = vbuf_format_fallbacks[i].from;
296 unsigned comp_bits = util_format_get_component_bits(format, 0, 0);
297
298 if ((comp_bits > 32) && !needs64b)
299 continue;
300
301 if (!screen->is_format_supported(screen, format, PIPE_BUFFER, 0, 0,
302 PIPE_BIND_VERTEX_BUFFER)) {
303 caps->format_translation[format] = vbuf_format_fallbacks[i].to;
304 caps->fallback_always = true;
305 }
306 }
307
308 caps->buffer_offset_unaligned =
309 !screen->get_param(screen,
310 PIPE_CAP_VERTEX_BUFFER_OFFSET_4BYTE_ALIGNED_ONLY);
311 caps->buffer_stride_unaligned =
312 !screen->get_param(screen,
313 PIPE_CAP_VERTEX_BUFFER_STRIDE_4BYTE_ALIGNED_ONLY);
314 caps->velem_src_offset_unaligned =
315 !screen->get_param(screen,
316 PIPE_CAP_VERTEX_ELEMENT_SRC_OFFSET_4BYTE_ALIGNED_ONLY);
317 caps->attrib_component_unaligned =
318 !screen->get_param(screen,
319 PIPE_CAP_VERTEX_ATTRIB_ELEMENT_ALIGNED_ONLY);
320 assert(caps->attrib_component_unaligned ||
321 (caps->velem_src_offset_unaligned && caps->buffer_stride_unaligned && caps->buffer_offset_unaligned));
322 caps->user_vertex_buffers =
323 screen->get_param(screen, PIPE_CAP_USER_VERTEX_BUFFERS);
324 caps->max_vertex_buffers =
325 screen->get_param(screen, PIPE_CAP_MAX_VERTEX_BUFFERS);
326
327 if (screen->get_param(screen, PIPE_CAP_PRIMITIVE_RESTART) ||
328 screen->get_param(screen, PIPE_CAP_PRIMITIVE_RESTART_FIXED_INDEX)) {
329 caps->rewrite_restart_index = screen->get_param(screen, PIPE_CAP_EMULATE_NONFIXED_PRIMITIVE_RESTART);
330 caps->supported_restart_modes = screen->get_param(screen, PIPE_CAP_SUPPORTED_PRIM_MODES_WITH_RESTART);
331 caps->supported_restart_modes |= BITFIELD_BIT(MESA_PRIM_PATCHES);
332 if (caps->supported_restart_modes != BITFIELD_MASK(MESA_PRIM_COUNT))
333 caps->fallback_always = true;
334 caps->fallback_always |= caps->rewrite_restart_index;
335 }
336 caps->supported_prim_modes = screen->get_param(screen, PIPE_CAP_SUPPORTED_PRIM_MODES);
337 if (caps->supported_prim_modes != BITFIELD_MASK(MESA_PRIM_COUNT))
338 caps->fallback_always = true;
339
340 if (!screen->is_format_supported(screen, PIPE_FORMAT_R8_UINT, PIPE_BUFFER, 0, 0, PIPE_BIND_INDEX_BUFFER))
341 caps->fallback_always = caps->rewrite_ubyte_ibs = true;
342
343 /* OpenGL 2.0 requires a minimum of 16 vertex buffers */
344 if (caps->max_vertex_buffers < 16)
345 caps->fallback_always = true;
346
347 if (!caps->buffer_offset_unaligned ||
348 !caps->buffer_stride_unaligned ||
349 !caps->attrib_component_unaligned ||
350 !caps->velem_src_offset_unaligned)
351 caps->fallback_always = true;
352
353 if (!caps->fallback_always && !caps->user_vertex_buffers)
354 caps->fallback_only_for_user_vbuffers = true;
355 }
356
357 struct u_vbuf *
u_vbuf_create(struct pipe_context * pipe,struct u_vbuf_caps * caps)358 u_vbuf_create(struct pipe_context *pipe, struct u_vbuf_caps *caps)
359 {
360 struct u_vbuf *mgr = CALLOC_STRUCT(u_vbuf);
361
362 mgr->caps = *caps;
363 mgr->pipe = pipe;
364 if (caps->rewrite_ubyte_ibs || caps->rewrite_restart_index ||
365 /* require all but patches */
366 ((caps->supported_prim_modes & caps->supported_restart_modes & BITFIELD_MASK(MESA_PRIM_COUNT))) !=
367 BITFIELD_MASK(MESA_PRIM_COUNT)) {
368 struct primconvert_config cfg;
369 cfg.fixed_prim_restart = caps->rewrite_restart_index;
370 cfg.primtypes_mask = caps->supported_prim_modes;
371 cfg.restart_primtypes_mask = caps->supported_restart_modes;
372 mgr->pc = util_primconvert_create_config(pipe, &cfg);
373 }
374 mgr->translate_cache = translate_cache_create();
375 memset(mgr->fallback_vbs, ~0, sizeof(mgr->fallback_vbs));
376 mgr->allowed_vb_mask = u_bit_consecutive(0, mgr->caps.max_vertex_buffers);
377
378 mgr->has_signed_vb_offset =
379 pipe->screen->get_param(pipe->screen,
380 PIPE_CAP_SIGNED_VERTEX_BUFFER_OFFSET);
381
382 cso_cache_init(&mgr->cso_cache, pipe);
383 cso_cache_set_delete_cso_callback(&mgr->cso_cache,
384 u_vbuf_delete_vertex_elements, pipe);
385
386 return mgr;
387 }
388
389 /* u_vbuf uses its own caching for vertex elements, because it needs to keep
390 * its own preprocessed state per vertex element CSO. */
391 static struct u_vbuf_elements *
u_vbuf_set_vertex_elements_internal(struct u_vbuf * mgr,const struct cso_velems_state * velems)392 u_vbuf_set_vertex_elements_internal(struct u_vbuf *mgr,
393 const struct cso_velems_state *velems)
394 {
395 struct pipe_context *pipe = mgr->pipe;
396 unsigned key_size, hash_key;
397 struct cso_hash_iter iter;
398 struct u_vbuf_elements *ve;
399
400 /* need to include the count into the stored state data too. */
401 key_size = sizeof(struct pipe_vertex_element) * velems->count +
402 sizeof(unsigned);
403 hash_key = cso_construct_key(velems, key_size);
404 iter = cso_find_state_template(&mgr->cso_cache, hash_key, CSO_VELEMENTS,
405 velems, key_size);
406
407 if (cso_hash_iter_is_null(iter)) {
408 struct cso_velements *cso = MALLOC_STRUCT(cso_velements);
409 memcpy(&cso->state, velems, key_size);
410 cso->data = u_vbuf_create_vertex_elements(mgr, velems->count,
411 velems->velems);
412
413 iter = cso_insert_state(&mgr->cso_cache, hash_key, CSO_VELEMENTS, cso);
414 ve = cso->data;
415 } else {
416 ve = ((struct cso_velements *)cso_hash_iter_data(iter))->data;
417 }
418
419 assert(ve);
420
421 if (ve != mgr->ve)
422 pipe->bind_vertex_elements_state(pipe, ve->driver_cso);
423
424 return ve;
425 }
426
u_vbuf_set_vertex_elements(struct u_vbuf * mgr,const struct cso_velems_state * velems)427 void u_vbuf_set_vertex_elements(struct u_vbuf *mgr,
428 const struct cso_velems_state *velems)
429 {
430 mgr->ve = u_vbuf_set_vertex_elements_internal(mgr, velems);
431 }
432
u_vbuf_set_flatshade_first(struct u_vbuf * mgr,bool flatshade_first)433 void u_vbuf_set_flatshade_first(struct u_vbuf *mgr, bool flatshade_first)
434 {
435 mgr->flatshade_first = flatshade_first;
436 }
437
u_vbuf_unset_vertex_elements(struct u_vbuf * mgr)438 void u_vbuf_unset_vertex_elements(struct u_vbuf *mgr)
439 {
440 mgr->ve = NULL;
441 }
442
u_vbuf_destroy(struct u_vbuf * mgr)443 void u_vbuf_destroy(struct u_vbuf *mgr)
444 {
445 unsigned i;
446
447 mgr->pipe->set_vertex_buffers(mgr->pipe, 0, NULL);
448
449 for (i = 0; i < PIPE_MAX_ATTRIBS; i++)
450 pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);
451 for (i = 0; i < PIPE_MAX_ATTRIBS; i++)
452 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);
453
454 if (mgr->pc)
455 util_primconvert_destroy(mgr->pc);
456
457 translate_cache_destroy(mgr->translate_cache);
458 cso_cache_delete(&mgr->cso_cache);
459 FREE(mgr);
460 }
461
462 static enum pipe_error
u_vbuf_translate_buffers(struct u_vbuf * mgr,struct translate_key * key,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw,unsigned vb_mask,unsigned out_vb,int start_vertex,unsigned num_vertices,int min_index,bool unroll_indices)463 u_vbuf_translate_buffers(struct u_vbuf *mgr, struct translate_key *key,
464 const struct pipe_draw_info *info,
465 const struct pipe_draw_start_count_bias *draw,
466 unsigned vb_mask, unsigned out_vb,
467 int start_vertex, unsigned num_vertices,
468 int min_index, bool unroll_indices)
469 {
470 struct translate *tr;
471 struct pipe_transfer *vb_transfer[PIPE_MAX_ATTRIBS] = {0};
472 struct pipe_resource *out_buffer = NULL;
473 uint8_t *out_map;
474 unsigned out_offset, mask;
475
476 /* Get a translate object. */
477 tr = translate_cache_find(mgr->translate_cache, key);
478
479 /* Map buffers we want to translate. */
480 mask = vb_mask;
481 while (mask) {
482 struct pipe_vertex_buffer *vb;
483 unsigned offset;
484 uint8_t *map;
485 unsigned i = u_bit_scan(&mask);
486 unsigned stride = mgr->ve->strides[i];
487
488 vb = &mgr->vertex_buffer[i];
489 offset = vb->buffer_offset + stride * start_vertex;
490
491 if (vb->is_user_buffer) {
492 map = (uint8_t*)vb->buffer.user + offset;
493 } else {
494 unsigned size = stride ? num_vertices * stride
495 : sizeof(double)*4;
496
497 if (!vb->buffer.resource) {
498 static uint64_t dummy_buf[4] = { 0 };
499 tr->set_buffer(tr, i, dummy_buf, 0, 0);
500 continue;
501 }
502
503 if (stride) {
504 /* the stride cannot be used to calculate the map size of the buffer,
505 * as it only determines the bytes between elements, not the size of elements
506 * themselves, meaning that if stride < element_size, the mapped size will
507 * be too small and conversion will overrun the map buffer
508 *
509 * instead, add the size of the largest possible attribute to the final attribute's offset
510 * in order to ensure the map is large enough
511 */
512 unsigned last_offset = size - stride;
513 size = MAX2(size, last_offset + sizeof(double)*4);
514 }
515
516 if (offset + size > vb->buffer.resource->width0) {
517 /* Don't try to map past end of buffer. This often happens when
518 * we're translating an attribute that's at offset > 0 from the
519 * start of the vertex. If we'd subtract attrib's offset from
520 * the size, this probably wouldn't happen.
521 */
522 size = vb->buffer.resource->width0 - offset;
523
524 /* Also adjust num_vertices. A common user error is to call
525 * glDrawRangeElements() with incorrect 'end' argument. The 'end
526 * value should be the max index value, but people often
527 * accidentally add one to this value. This adjustment avoids
528 * crashing (by reading past the end of a hardware buffer mapping)
529 * when people do that.
530 */
531 num_vertices = (size + stride - 1) / stride;
532 }
533
534 map = pipe_buffer_map_range(mgr->pipe, vb->buffer.resource, offset, size,
535 PIPE_MAP_READ, &vb_transfer[i]);
536 }
537
538 /* Subtract min_index so that indexing with the index buffer works. */
539 if (unroll_indices) {
540 map -= (ptrdiff_t)stride * min_index;
541 }
542
543 tr->set_buffer(tr, i, map, stride, info->max_index);
544 }
545
546 /* Translate. */
547 if (unroll_indices) {
548 struct pipe_transfer *transfer = NULL;
549 const unsigned offset = draw->start * info->index_size;
550 uint8_t *map;
551
552 /* Create and map the output buffer. */
553 u_upload_alloc(mgr->pipe->stream_uploader, 0,
554 key->output_stride * draw->count, 4,
555 &out_offset, &out_buffer,
556 (void**)&out_map);
557 if (!out_buffer)
558 return PIPE_ERROR_OUT_OF_MEMORY;
559
560 if (info->has_user_indices) {
561 map = (uint8_t*)info->index.user + offset;
562 } else {
563 map = pipe_buffer_map_range(mgr->pipe, info->index.resource, offset,
564 draw->count * info->index_size,
565 PIPE_MAP_READ, &transfer);
566 }
567
568 switch (info->index_size) {
569 case 4:
570 tr->run_elts(tr, (unsigned*)map, draw->count, 0, 0, out_map);
571 break;
572 case 2:
573 tr->run_elts16(tr, (uint16_t*)map, draw->count, 0, 0, out_map);
574 break;
575 case 1:
576 tr->run_elts8(tr, map, draw->count, 0, 0, out_map);
577 break;
578 }
579
580 if (transfer) {
581 pipe_buffer_unmap(mgr->pipe, transfer);
582 }
583 } else {
584 /* Create and map the output buffer. */
585 u_upload_alloc(mgr->pipe->stream_uploader,
586 mgr->has_signed_vb_offset ?
587 0 : key->output_stride * start_vertex,
588 key->output_stride * num_vertices, 4,
589 &out_offset, &out_buffer,
590 (void**)&out_map);
591 if (!out_buffer)
592 return PIPE_ERROR_OUT_OF_MEMORY;
593
594 out_offset -= key->output_stride * start_vertex;
595
596 tr->run(tr, 0, num_vertices, 0, 0, out_map);
597 }
598
599 /* Unmap all buffers. */
600 mask = vb_mask;
601 while (mask) {
602 unsigned i = u_bit_scan(&mask);
603
604 if (vb_transfer[i]) {
605 pipe_buffer_unmap(mgr->pipe, vb_transfer[i]);
606 }
607 }
608
609 /* Setup the new vertex buffer. */
610 mgr->real_vertex_buffer[out_vb].buffer_offset = out_offset;
611
612 /* Move the buffer reference. */
613 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[out_vb]);
614 mgr->real_vertex_buffer[out_vb].buffer.resource = out_buffer;
615 mgr->real_vertex_buffer[out_vb].is_user_buffer = false;
616
617 return PIPE_OK;
618 }
619
620 static bool
u_vbuf_translate_find_free_vb_slots(struct u_vbuf * mgr,unsigned mask[VB_NUM])621 u_vbuf_translate_find_free_vb_slots(struct u_vbuf *mgr,
622 unsigned mask[VB_NUM])
623 {
624 unsigned type;
625 unsigned fallback_vbs[VB_NUM];
626 /* Set the bit for each buffer which is incompatible, or isn't set. */
627 uint32_t unused_vb_mask =
628 mgr->ve->incompatible_vb_mask_all | mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask |
629 ~mgr->enabled_vb_mask;
630 uint32_t unused_vb_mask_orig;
631 bool insufficient_buffers = false;
632
633 /* No vertex buffers available at all */
634 if (!unused_vb_mask)
635 return false;
636
637 memset(fallback_vbs, ~0, sizeof(fallback_vbs));
638 mgr->fallback_vbs_mask = 0;
639
640 /* Find free slots for each type if needed. */
641 unused_vb_mask_orig = unused_vb_mask;
642 for (type = 0; type < VB_NUM; type++) {
643 if (mask[type]) {
644 uint32_t index;
645
646 if (!unused_vb_mask) {
647 insufficient_buffers = true;
648 break;
649 }
650
651 index = ffs(unused_vb_mask) - 1;
652 fallback_vbs[type] = index;
653 mgr->fallback_vbs_mask |= 1 << index;
654 unused_vb_mask &= ~(1 << index);
655 /*printf("found slot=%i for type=%i\n", index, type);*/
656 }
657 }
658
659 if (insufficient_buffers) {
660 /* not enough vbs for all types supported by the hardware, they will have to share one
661 * buffer */
662 uint32_t index = ffs(unused_vb_mask_orig) - 1;
663 /* When sharing one vertex buffer use per-vertex frequency for everything. */
664 fallback_vbs[VB_VERTEX] = index;
665 mgr->fallback_vbs_mask = 1 << index;
666 mask[VB_VERTEX] = mask[VB_VERTEX] | mask[VB_CONST] | mask[VB_INSTANCE];
667 mask[VB_CONST] = 0;
668 mask[VB_INSTANCE] = 0;
669 }
670
671 for (type = 0; type < VB_NUM; type++) {
672 if (mask[type]) {
673 mgr->num_real_vertex_buffers =
674 MAX2(mgr->num_real_vertex_buffers, fallback_vbs[type] + 1);
675 mgr->vertex_buffers_dirty = true;
676 }
677 }
678
679 memcpy(mgr->fallback_vbs, fallback_vbs, sizeof(fallback_vbs));
680 return true;
681 }
682
683 static bool
u_vbuf_translate_begin(struct u_vbuf * mgr,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw,int start_vertex,unsigned num_vertices,int min_index,bool unroll_indices,uint32_t misaligned)684 u_vbuf_translate_begin(struct u_vbuf *mgr,
685 const struct pipe_draw_info *info,
686 const struct pipe_draw_start_count_bias *draw,
687 int start_vertex, unsigned num_vertices,
688 int min_index, bool unroll_indices,
689 uint32_t misaligned)
690 {
691 unsigned mask[VB_NUM] = {0};
692 struct translate_key key[VB_NUM];
693 unsigned elem_index[VB_NUM][PIPE_MAX_ATTRIBS]; /* ... into key.elements */
694 unsigned i, type;
695 const unsigned incompatible_vb_mask = (misaligned | mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask) &
696 mgr->ve->used_vb_mask;
697
698 const int start[VB_NUM] = {
699 start_vertex, /* VERTEX */
700 info->start_instance, /* INSTANCE */
701 0 /* CONST */
702 };
703
704 const unsigned num[VB_NUM] = {
705 num_vertices, /* VERTEX */
706 info->instance_count, /* INSTANCE */
707 1 /* CONST */
708 };
709
710 memset(key, 0, sizeof(key));
711 memset(elem_index, ~0, sizeof(elem_index));
712
713 /* See if there are vertex attribs of each type to translate and
714 * which ones. */
715 for (i = 0; i < mgr->ve->count; i++) {
716 unsigned vb_index = mgr->ve->ve[i].vertex_buffer_index;
717
718 if (!mgr->ve->ve[i].src_stride) {
719 if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&
720 !(incompatible_vb_mask & (1 << vb_index))) {
721 continue;
722 }
723 mask[VB_CONST] |= 1 << vb_index;
724 } else if (mgr->ve->ve[i].instance_divisor) {
725 if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&
726 !(incompatible_vb_mask & (1 << vb_index))) {
727 continue;
728 }
729 mask[VB_INSTANCE] |= 1 << vb_index;
730 } else {
731 if (!unroll_indices &&
732 !(mgr->ve->incompatible_elem_mask & (1 << i)) &&
733 !(incompatible_vb_mask & (1 << vb_index))) {
734 continue;
735 }
736 mask[VB_VERTEX] |= 1 << vb_index;
737 }
738 }
739
740 assert(mask[VB_VERTEX] || mask[VB_INSTANCE] || mask[VB_CONST]);
741
742 /* Find free vertex buffer slots. */
743 if (!u_vbuf_translate_find_free_vb_slots(mgr, mask)) {
744 return false;
745 }
746
747 unsigned min_alignment[VB_NUM] = {0};
748 /* Initialize the translate keys. */
749 for (i = 0; i < mgr->ve->count; i++) {
750 struct translate_key *k;
751 struct translate_element *te;
752 enum pipe_format output_format = mgr->ve->native_format[i];
753 unsigned bit, vb_index = mgr->ve->ve[i].vertex_buffer_index;
754 bit = 1 << vb_index;
755
756 if (!(mgr->ve->incompatible_elem_mask & (1 << i)) &&
757 !(incompatible_vb_mask & (1 << vb_index)) &&
758 (!unroll_indices || !(mask[VB_VERTEX] & bit))) {
759 continue;
760 }
761
762 /* Set type to what we will translate.
763 * Whether vertex, instance, or constant attribs. */
764 for (type = 0; type < VB_NUM; type++) {
765 if (mask[type] & bit) {
766 break;
767 }
768 }
769 assert(type < VB_NUM);
770 if (mgr->ve->ve[i].src_format != output_format)
771 assert(translate_is_output_format_supported(output_format));
772 /*printf("velem=%i type=%i\n", i, type);*/
773
774 /* Add the vertex element. */
775 k = &key[type];
776 elem_index[type][i] = k->nr_elements;
777
778 te = &k->element[k->nr_elements];
779 te->type = TRANSLATE_ELEMENT_NORMAL;
780 te->instance_divisor = 0;
781 te->input_buffer = vb_index;
782 te->input_format = mgr->ve->ve[i].src_format;
783 te->input_offset = mgr->ve->ve[i].src_offset;
784 te->output_format = output_format;
785 te->output_offset = k->output_stride;
786 unsigned adjustment = 0;
787 if (!mgr->caps.attrib_component_unaligned &&
788 te->output_offset % mgr->ve->component_size[i] != 0) {
789 unsigned aligned = align(te->output_offset, mgr->ve->component_size[i]);
790 adjustment = aligned - te->output_offset;
791 te->output_offset = aligned;
792 }
793
794 k->output_stride += mgr->ve->native_format_size[i] + adjustment;
795 k->nr_elements++;
796 min_alignment[type] = MAX2(min_alignment[type], mgr->ve->component_size[i]);
797 }
798
799 /* Translate buffers. */
800 for (type = 0; type < VB_NUM; type++) {
801 if (key[type].nr_elements) {
802 enum pipe_error err;
803 if (!mgr->caps.attrib_component_unaligned)
804 key[type].output_stride = align(key[type].output_stride, min_alignment[type]);
805 err = u_vbuf_translate_buffers(mgr, &key[type], info, draw,
806 mask[type], mgr->fallback_vbs[type],
807 start[type], num[type], min_index,
808 unroll_indices && type == VB_VERTEX);
809 if (err != PIPE_OK)
810 return false;
811 }
812 }
813
814 /* Setup new vertex elements. */
815 for (i = 0; i < mgr->ve->count; i++) {
816 for (type = 0; type < VB_NUM; type++) {
817 if (elem_index[type][i] < key[type].nr_elements) {
818 struct translate_element *te = &key[type].element[elem_index[type][i]];
819 mgr->fallback_velems.velems[i].instance_divisor = mgr->ve->ve[i].instance_divisor;
820 mgr->fallback_velems.velems[i].src_format = te->output_format;
821 mgr->fallback_velems.velems[i].src_offset = te->output_offset;
822 mgr->fallback_velems.velems[i].vertex_buffer_index = mgr->fallback_vbs[type];
823
824 /* Fixup the stride for constant attribs. */
825 if (type == VB_CONST)
826 mgr->fallback_velems.velems[i].src_stride = 0;
827 else
828 mgr->fallback_velems.velems[i].src_stride = key[type].output_stride;
829
830 /* elem_index[type][i] can only be set for one type. */
831 assert(type > VB_INSTANCE || elem_index[type+1][i] == ~0u);
832 assert(type > VB_VERTEX || elem_index[type+2][i] == ~0u);
833 break;
834 }
835 }
836 /* No translating, just copy the original vertex element over. */
837 if (type == VB_NUM) {
838 memcpy(&mgr->fallback_velems.velems[i], &mgr->ve->ve[i],
839 sizeof(struct pipe_vertex_element));
840 }
841 }
842
843 mgr->fallback_velems.count = mgr->ve->count;
844
845 u_vbuf_set_vertex_elements_internal(mgr, &mgr->fallback_velems);
846 mgr->using_translate = true;
847 return true;
848 }
849
u_vbuf_translate_end(struct u_vbuf * mgr)850 static void u_vbuf_translate_end(struct u_vbuf *mgr)
851 {
852 unsigned i;
853
854 /* Restore vertex elements. */
855 mgr->pipe->bind_vertex_elements_state(mgr->pipe, mgr->ve->driver_cso);
856 mgr->using_translate = false;
857
858 /* Unreference the now-unused VBOs. */
859 for (i = 0; i < VB_NUM; i++) {
860 unsigned vb = mgr->fallback_vbs[i];
861 if (vb != ~0u) {
862 pipe_resource_reference(&mgr->real_vertex_buffer[vb].buffer.resource, NULL);
863 mgr->fallback_vbs[i] = ~0;
864 }
865 }
866 /* This will cause the fallback buffers above num_vertex_buffers to be
867 * unbound.
868 */
869 mgr->num_real_vertex_buffers = mgr->num_vertex_buffers;
870 mgr->vertex_buffers_dirty = true;
871 mgr->fallback_vbs_mask = 0;
872 }
873
874 static void *
u_vbuf_create_vertex_elements(struct u_vbuf * mgr,unsigned count,const struct pipe_vertex_element * attribs)875 u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
876 const struct pipe_vertex_element *attribs)
877 {
878 struct pipe_vertex_element tmp[PIPE_MAX_ATTRIBS];
879 util_lower_uint64_vertex_elements(&attribs, &count, tmp);
880
881 struct pipe_context *pipe = mgr->pipe;
882 unsigned i;
883 struct pipe_vertex_element driver_attribs[PIPE_MAX_ATTRIBS];
884 struct u_vbuf_elements *ve = CALLOC_STRUCT(u_vbuf_elements);
885 uint32_t used_buffers = 0;
886
887 ve->count = count;
888
889 memcpy(ve->ve, attribs, sizeof(struct pipe_vertex_element) * count);
890 memcpy(driver_attribs, attribs, sizeof(struct pipe_vertex_element) * count);
891
892 /* Set the best native format in case the original format is not
893 * supported. */
894 for (i = 0; i < count; i++) {
895 enum pipe_format format = ve->ve[i].src_format;
896 unsigned vb_index_bit = 1 << ve->ve[i].vertex_buffer_index;
897
898 ve->src_format_size[i] = util_format_get_blocksize(format);
899
900 if (used_buffers & vb_index_bit)
901 ve->interleaved_vb_mask |= vb_index_bit;
902
903 used_buffers |= vb_index_bit;
904
905 if (!ve->ve[i].instance_divisor) {
906 ve->noninstance_vb_mask_any |= vb_index_bit;
907 }
908
909 format = mgr->caps.format_translation[format];
910
911 driver_attribs[i].src_format = format;
912 ve->native_format[i] = format;
913 ve->native_format_size[i] =
914 util_format_get_blocksize(ve->native_format[i]);
915
916 const struct util_format_description *desc = util_format_description(format);
917 bool is_packed = false;
918 for (unsigned c = 0; c < desc->nr_channels; c++)
919 is_packed |= desc->channel[c].size != desc->channel[0].size || desc->channel[c].size % 8 != 0;
920 unsigned component_size = is_packed ?
921 ve->native_format_size[i] : (ve->native_format_size[i] / desc->nr_channels);
922 ve->component_size[i] = component_size;
923
924 if (ve->ve[i].src_format != format ||
925 (!mgr->caps.velem_src_offset_unaligned &&
926 ve->ve[i].src_offset % 4 != 0) ||
927 (!mgr->caps.attrib_component_unaligned &&
928 ve->ve[i].src_offset % component_size != 0)) {
929 ve->incompatible_elem_mask |= 1 << i;
930 ve->incompatible_vb_mask_any |= vb_index_bit;
931 } else {
932 ve->compatible_vb_mask_any |= vb_index_bit;
933 if (component_size == 2) {
934 ve->vb_align_mask[0] |= vb_index_bit;
935 if (ve->ve[i].src_stride % 2 != 0)
936 ve->incompatible_vb_mask |= vb_index_bit;
937 }
938 else if (component_size == 4) {
939 ve->vb_align_mask[1] |= vb_index_bit;
940 if (ve->ve[i].src_stride % 4 != 0)
941 ve->incompatible_vb_mask |= vb_index_bit;
942 }
943 }
944 ve->strides[ve->ve[i].vertex_buffer_index] = ve->ve[i].src_stride;
945 if (ve->ve[i].src_stride) {
946 ve->nonzero_stride_vb_mask |= 1 << ve->ve[i].vertex_buffer_index;
947 }
948 if (!mgr->caps.buffer_stride_unaligned && ve->ve[i].src_stride % 4 != 0)
949 ve->incompatible_vb_mask |= vb_index_bit;
950 }
951
952 if (used_buffers & ~mgr->allowed_vb_mask) {
953 /* More vertex buffers are used than the hardware supports. In
954 * principle, we only need to make sure that less vertex buffers are
955 * used, and mark some of the latter vertex buffers as incompatible.
956 * For now, mark all vertex buffers as incompatible.
957 */
958 ve->incompatible_vb_mask_any = used_buffers;
959 ve->compatible_vb_mask_any = 0;
960 ve->incompatible_elem_mask = u_bit_consecutive(0, count);
961 }
962
963 ve->used_vb_mask = used_buffers;
964 ve->compatible_vb_mask_all = ~ve->incompatible_vb_mask_any & used_buffers;
965 ve->incompatible_vb_mask_all = ~ve->compatible_vb_mask_any & used_buffers;
966
967 /* Align the formats and offsets to the size of DWORD if needed. */
968 if (!mgr->caps.velem_src_offset_unaligned) {
969 for (i = 0; i < count; i++) {
970 ve->native_format_size[i] = align(ve->native_format_size[i], 4);
971 driver_attribs[i].src_offset = align(ve->ve[i].src_offset, 4);
972 }
973 }
974
975 /* Only create driver CSO if no incompatible elements */
976 if (!ve->incompatible_elem_mask) {
977 ve->driver_cso =
978 pipe->create_vertex_elements_state(pipe, count, driver_attribs);
979 }
980
981 return ve;
982 }
983
u_vbuf_delete_vertex_elements(void * ctx,void * state,enum cso_cache_type type)984 static void u_vbuf_delete_vertex_elements(void *ctx, void *state,
985 enum cso_cache_type type)
986 {
987 struct pipe_context *pipe = (struct pipe_context*)ctx;
988 struct cso_velements *cso = (struct cso_velements*)state;
989 struct u_vbuf_elements *ve = (struct u_vbuf_elements*)cso->data;
990
991 if (ve->driver_cso)
992 pipe->delete_vertex_elements_state(pipe, ve->driver_cso);
993 FREE(ve);
994 FREE(cso);
995 }
996
u_vbuf_set_vertex_buffers(struct u_vbuf * mgr,unsigned count,bool take_ownership,const struct pipe_vertex_buffer * bufs)997 void u_vbuf_set_vertex_buffers(struct u_vbuf *mgr,
998 unsigned count,
999 bool take_ownership,
1000 const struct pipe_vertex_buffer *bufs)
1001 {
1002 if (!count) {
1003 struct pipe_context *pipe = mgr->pipe;
1004 unsigned last_count = mgr->num_vertex_buffers;
1005
1006 /* Unbind. */
1007 mgr->num_vertex_buffers = 0;
1008 mgr->num_real_vertex_buffers = 0;
1009 mgr->user_vb_mask = 0;
1010 mgr->incompatible_vb_mask = 0;
1011 mgr->enabled_vb_mask = 0;
1012 mgr->unaligned_vb_mask[0] = 0;
1013 mgr->unaligned_vb_mask[1] = 0;
1014 mgr->vertex_buffers_dirty = false;
1015
1016 for (unsigned i = 0; i < last_count; i++) {
1017 pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);
1018 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);
1019 }
1020
1021 pipe->set_vertex_buffers(pipe, 0, NULL);
1022 return;
1023 }
1024
1025 assert(bufs);
1026
1027 unsigned i;
1028 /* which buffers are enabled */
1029 uint32_t enabled_vb_mask = 0;
1030 /* which buffers are in user memory */
1031 uint32_t user_vb_mask = 0;
1032 /* which buffers are incompatible with the driver */
1033 uint32_t incompatible_vb_mask = 0;
1034 /* which buffers are unaligned to 2/4 bytes */
1035 uint32_t unaligned_vb_mask[2] = {0};
1036 unsigned num_identical = 0;
1037
1038 for (i = 0; i < count; i++) {
1039 const struct pipe_vertex_buffer *vb = &bufs[i];
1040 struct pipe_vertex_buffer *orig_vb = &mgr->vertex_buffer[i];
1041 struct pipe_vertex_buffer *real_vb = &mgr->real_vertex_buffer[i];
1042
1043 if (!vb->buffer.resource) {
1044 pipe_vertex_buffer_unreference(orig_vb);
1045 pipe_vertex_buffer_unreference(real_vb);
1046 continue;
1047 }
1048
1049 /* The structure has holes: do not use memcmp. */
1050 if (orig_vb->is_user_buffer == vb->is_user_buffer &&
1051 orig_vb->buffer_offset == vb->buffer_offset &&
1052 orig_vb->buffer.resource == vb->buffer.resource)
1053 num_identical++;
1054
1055 if (take_ownership) {
1056 pipe_vertex_buffer_unreference(orig_vb);
1057 memcpy(orig_vb, vb, sizeof(*vb));
1058 } else {
1059 pipe_vertex_buffer_reference(orig_vb, vb);
1060 }
1061
1062 enabled_vb_mask |= 1 << i;
1063
1064 if ((!mgr->caps.buffer_offset_unaligned && vb->buffer_offset % 4 != 0)) {
1065 incompatible_vb_mask |= 1 << i;
1066 real_vb->buffer_offset = vb->buffer_offset;
1067 pipe_vertex_buffer_unreference(real_vb);
1068 real_vb->is_user_buffer = false;
1069 continue;
1070 }
1071
1072 if (!mgr->caps.attrib_component_unaligned) {
1073 if (vb->buffer_offset % 2 != 0)
1074 unaligned_vb_mask[0] |= BITFIELD_BIT(i);
1075 if (vb->buffer_offset % 4 != 0)
1076 unaligned_vb_mask[1] |= BITFIELD_BIT(i);
1077 }
1078
1079 if (!mgr->caps.user_vertex_buffers && vb->is_user_buffer) {
1080 user_vb_mask |= 1 << i;
1081 real_vb->buffer_offset = vb->buffer_offset;
1082 pipe_vertex_buffer_unreference(real_vb);
1083 real_vb->is_user_buffer = false;
1084 continue;
1085 }
1086
1087 pipe_vertex_buffer_reference(real_vb, vb);
1088 }
1089
1090 unsigned last_count = mgr->num_vertex_buffers;
1091
1092 if (num_identical == count && count == last_count)
1093 return;
1094
1095 for (; i < last_count; i++) {
1096 pipe_vertex_buffer_unreference(&mgr->vertex_buffer[i]);
1097 pipe_vertex_buffer_unreference(&mgr->real_vertex_buffer[i]);
1098 }
1099
1100 mgr->num_vertex_buffers = count;
1101 mgr->num_real_vertex_buffers = count;
1102 mgr->user_vb_mask = user_vb_mask;
1103 mgr->incompatible_vb_mask = incompatible_vb_mask;
1104 mgr->enabled_vb_mask = enabled_vb_mask;
1105 mgr->unaligned_vb_mask[0] = unaligned_vb_mask[0];
1106 mgr->unaligned_vb_mask[1] = unaligned_vb_mask[1];
1107 mgr->vertex_buffers_dirty = true;
1108 }
1109
1110 static ALWAYS_INLINE bool
get_upload_offset_size(struct u_vbuf * mgr,const struct pipe_vertex_buffer * vb,struct u_vbuf_elements * ve,const struct pipe_vertex_element * velem,unsigned vb_index,unsigned velem_index,int start_vertex,unsigned num_vertices,int start_instance,unsigned num_instances,unsigned * offset,unsigned * size)1111 get_upload_offset_size(struct u_vbuf *mgr,
1112 const struct pipe_vertex_buffer *vb,
1113 struct u_vbuf_elements *ve,
1114 const struct pipe_vertex_element *velem,
1115 unsigned vb_index, unsigned velem_index,
1116 int start_vertex, unsigned num_vertices,
1117 int start_instance, unsigned num_instances,
1118 unsigned *offset, unsigned *size)
1119 {
1120 /* Skip the buffers generated by translate. */
1121 if ((1 << vb_index) & mgr->fallback_vbs_mask || !vb->is_user_buffer)
1122 return false;
1123
1124 unsigned instance_div = velem->instance_divisor;
1125 *offset = vb->buffer_offset + velem->src_offset;
1126
1127 if (!velem->src_stride) {
1128 /* Constant attrib. */
1129 *size = ve->src_format_size[velem_index];
1130 } else if (instance_div) {
1131 /* Per-instance attrib. */
1132
1133 /* Figure out how many instances we'll render given instance_div. We
1134 * can't use the typical div_round_up() pattern because the CTS uses
1135 * instance_div = ~0 for a test, which overflows div_round_up()'s
1136 * addition.
1137 */
1138 unsigned count = num_instances / instance_div;
1139 if (count * instance_div != num_instances)
1140 count++;
1141
1142 *offset += velem->src_stride * start_instance;
1143 *size = velem->src_stride * (count - 1) + ve->src_format_size[velem_index];
1144 } else {
1145 /* Per-vertex attrib. */
1146 *offset += velem->src_stride * start_vertex;
1147 *size = velem->src_stride * (num_vertices - 1) + ve->src_format_size[velem_index];
1148 }
1149 return true;
1150 }
1151
1152
1153 static enum pipe_error
u_vbuf_upload_buffers(struct u_vbuf * mgr,int start_vertex,unsigned num_vertices,int start_instance,unsigned num_instances)1154 u_vbuf_upload_buffers(struct u_vbuf *mgr,
1155 int start_vertex, unsigned num_vertices,
1156 int start_instance, unsigned num_instances)
1157 {
1158 unsigned i;
1159 struct u_vbuf_elements *ve = mgr->ve;
1160 unsigned nr_velems = ve->count;
1161 const struct pipe_vertex_element *velems =
1162 mgr->using_translate ? mgr->fallback_velems.velems : ve->ve;
1163
1164 /* Faster path when no vertex attribs are interleaved. */
1165 if ((ve->interleaved_vb_mask & mgr->user_vb_mask) == 0) {
1166 for (i = 0; i < nr_velems; i++) {
1167 const struct pipe_vertex_element *velem = &velems[i];
1168 unsigned index = velem->vertex_buffer_index;
1169 struct pipe_vertex_buffer *vb = &mgr->vertex_buffer[index];
1170 unsigned offset, size;
1171
1172 if (!get_upload_offset_size(mgr, vb, ve, velem, index, i, start_vertex,
1173 num_vertices, start_instance, num_instances,
1174 &offset, &size))
1175 continue;
1176
1177 struct pipe_vertex_buffer *real_vb = &mgr->real_vertex_buffer[index];
1178 const uint8_t *ptr = mgr->vertex_buffer[index].buffer.user;
1179
1180 u_upload_data(mgr->pipe->stream_uploader,
1181 mgr->has_signed_vb_offset ? 0 : offset,
1182 size, 4, ptr + offset, &real_vb->buffer_offset,
1183 &real_vb->buffer.resource);
1184 if (!real_vb->buffer.resource)
1185 return PIPE_ERROR_OUT_OF_MEMORY;
1186
1187 real_vb->buffer_offset -= offset;
1188 }
1189 return PIPE_OK;
1190 }
1191
1192 unsigned start_offset[PIPE_MAX_ATTRIBS];
1193 unsigned end_offset[PIPE_MAX_ATTRIBS];
1194 uint32_t buffer_mask = 0;
1195
1196 /* Slower path supporting interleaved vertex attribs using 2 loops. */
1197 /* Determine how much data needs to be uploaded. */
1198 for (i = 0; i < nr_velems; i++) {
1199 const struct pipe_vertex_element *velem = &velems[i];
1200 unsigned index = velem->vertex_buffer_index;
1201 struct pipe_vertex_buffer *vb = &mgr->vertex_buffer[index];
1202 unsigned first, size, index_bit;
1203
1204 if (!get_upload_offset_size(mgr, vb, ve, velem, index, i, start_vertex,
1205 num_vertices, start_instance, num_instances,
1206 &first, &size))
1207 continue;
1208
1209 index_bit = 1 << index;
1210
1211 /* Update offsets. */
1212 if (!(buffer_mask & index_bit)) {
1213 start_offset[index] = first;
1214 end_offset[index] = first + size;
1215 } else {
1216 if (first < start_offset[index])
1217 start_offset[index] = first;
1218 if (first + size > end_offset[index])
1219 end_offset[index] = first + size;
1220 }
1221
1222 buffer_mask |= index_bit;
1223 }
1224
1225 /* Upload buffers. */
1226 while (buffer_mask) {
1227 unsigned start, end;
1228 struct pipe_vertex_buffer *real_vb;
1229 const uint8_t *ptr;
1230
1231 i = u_bit_scan(&buffer_mask);
1232
1233 start = start_offset[i];
1234 end = end_offset[i];
1235 assert(start < end);
1236
1237 real_vb = &mgr->real_vertex_buffer[i];
1238 ptr = mgr->vertex_buffer[i].buffer.user;
1239
1240 u_upload_data(mgr->pipe->stream_uploader,
1241 mgr->has_signed_vb_offset ? 0 : start,
1242 end - start, 4,
1243 ptr + start, &real_vb->buffer_offset, &real_vb->buffer.resource);
1244 if (!real_vb->buffer.resource)
1245 return PIPE_ERROR_OUT_OF_MEMORY;
1246
1247 real_vb->buffer_offset -= start;
1248 }
1249
1250 return PIPE_OK;
1251 }
1252
u_vbuf_need_minmax_index(const struct u_vbuf * mgr,uint32_t misaligned)1253 static bool u_vbuf_need_minmax_index(const struct u_vbuf *mgr, uint32_t misaligned)
1254 {
1255 /* See if there are any per-vertex attribs which will be uploaded or
1256 * translated. Use bitmasks to get the info instead of looping over vertex
1257 * elements. */
1258 return (mgr->ve->used_vb_mask &
1259 ((mgr->user_vb_mask |
1260 mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask |
1261 misaligned |
1262 mgr->ve->incompatible_vb_mask_any) &
1263 mgr->ve->noninstance_vb_mask_any &
1264 mgr->ve->nonzero_stride_vb_mask)) != 0;
1265 }
1266
u_vbuf_mapping_vertex_buffer_blocks(const struct u_vbuf * mgr,uint32_t misaligned)1267 static bool u_vbuf_mapping_vertex_buffer_blocks(const struct u_vbuf *mgr, uint32_t misaligned)
1268 {
1269 /* Return true if there are hw buffers which don't need to be translated.
1270 *
1271 * We could query whether each buffer is busy, but that would
1272 * be way more costly than this. */
1273 return (mgr->ve->used_vb_mask &
1274 (~mgr->user_vb_mask &
1275 ~mgr->incompatible_vb_mask &
1276 ~mgr->ve->incompatible_vb_mask &
1277 ~misaligned &
1278 mgr->ve->compatible_vb_mask_all &
1279 mgr->ve->noninstance_vb_mask_any &
1280 mgr->ve->nonzero_stride_vb_mask)) != 0;
1281 }
1282
1283 static void
u_vbuf_get_minmax_index_mapped(const struct pipe_draw_info * info,unsigned count,const void * indices,unsigned * out_min_index,unsigned * out_max_index)1284 u_vbuf_get_minmax_index_mapped(const struct pipe_draw_info *info,
1285 unsigned count,
1286 const void *indices, unsigned *out_min_index,
1287 unsigned *out_max_index)
1288 {
1289 if (!count) {
1290 *out_min_index = 0;
1291 *out_max_index = 0;
1292 return;
1293 }
1294
1295 switch (info->index_size) {
1296 case 4: {
1297 const unsigned *ui_indices = (const unsigned*)indices;
1298 unsigned max = 0;
1299 unsigned min = ~0u;
1300 if (info->primitive_restart) {
1301 for (unsigned i = 0; i < count; i++) {
1302 if (ui_indices[i] != info->restart_index) {
1303 if (ui_indices[i] > max) max = ui_indices[i];
1304 if (ui_indices[i] < min) min = ui_indices[i];
1305 }
1306 }
1307 }
1308 else {
1309 for (unsigned i = 0; i < count; i++) {
1310 if (ui_indices[i] > max) max = ui_indices[i];
1311 if (ui_indices[i] < min) min = ui_indices[i];
1312 }
1313 }
1314 *out_min_index = min;
1315 *out_max_index = max;
1316 break;
1317 }
1318 case 2: {
1319 const unsigned short *us_indices = (const unsigned short*)indices;
1320 unsigned short max = 0;
1321 unsigned short min = ~((unsigned short)0);
1322 if (info->primitive_restart) {
1323 for (unsigned i = 0; i < count; i++) {
1324 if (us_indices[i] != info->restart_index) {
1325 if (us_indices[i] > max) max = us_indices[i];
1326 if (us_indices[i] < min) min = us_indices[i];
1327 }
1328 }
1329 }
1330 else {
1331 for (unsigned i = 0; i < count; i++) {
1332 if (us_indices[i] > max) max = us_indices[i];
1333 if (us_indices[i] < min) min = us_indices[i];
1334 }
1335 }
1336 *out_min_index = min;
1337 *out_max_index = max;
1338 break;
1339 }
1340 case 1: {
1341 const unsigned char *ub_indices = (const unsigned char*)indices;
1342 unsigned char max = 0;
1343 unsigned char min = ~((unsigned char)0);
1344 if (info->primitive_restart) {
1345 for (unsigned i = 0; i < count; i++) {
1346 if (ub_indices[i] != info->restart_index) {
1347 if (ub_indices[i] > max) max = ub_indices[i];
1348 if (ub_indices[i] < min) min = ub_indices[i];
1349 }
1350 }
1351 }
1352 else {
1353 for (unsigned i = 0; i < count; i++) {
1354 if (ub_indices[i] > max) max = ub_indices[i];
1355 if (ub_indices[i] < min) min = ub_indices[i];
1356 }
1357 }
1358 *out_min_index = min;
1359 *out_max_index = max;
1360 break;
1361 }
1362 default:
1363 unreachable("bad index size");
1364 }
1365 }
1366
u_vbuf_get_minmax_index(struct pipe_context * pipe,const struct pipe_draw_info * info,const struct pipe_draw_start_count_bias * draw,unsigned * out_min_index,unsigned * out_max_index)1367 void u_vbuf_get_minmax_index(struct pipe_context *pipe,
1368 const struct pipe_draw_info *info,
1369 const struct pipe_draw_start_count_bias *draw,
1370 unsigned *out_min_index, unsigned *out_max_index)
1371 {
1372 struct pipe_transfer *transfer = NULL;
1373 const void *indices;
1374
1375 if (info->has_user_indices) {
1376 indices = (uint8_t*)info->index.user +
1377 draw->start * info->index_size;
1378 } else {
1379 indices = pipe_buffer_map_range(pipe, info->index.resource,
1380 draw->start * info->index_size,
1381 draw->count * info->index_size,
1382 PIPE_MAP_READ, &transfer);
1383 }
1384
1385 u_vbuf_get_minmax_index_mapped(info, draw->count, indices,
1386 out_min_index, out_max_index);
1387
1388 if (transfer) {
1389 pipe_buffer_unmap(pipe, transfer);
1390 }
1391 }
1392
u_vbuf_set_driver_vertex_buffers(struct u_vbuf * mgr)1393 static void u_vbuf_set_driver_vertex_buffers(struct u_vbuf *mgr)
1394 {
1395 struct pipe_context *pipe = mgr->pipe;
1396 unsigned count = mgr->num_real_vertex_buffers;
1397
1398 assert(mgr->vertex_buffers_dirty);
1399
1400 if (mgr->user_vb_mask == BITFIELD_MASK(count)) {
1401 /* Fast path that allows us to transfer the VBO references to the driver
1402 * to skip atomic reference counting there. These are freshly uploaded
1403 * user buffers that can be discarded after this call.
1404 */
1405 pipe->set_vertex_buffers(pipe, count, mgr->real_vertex_buffer);
1406
1407 /* We don't own the VBO references now. Set them to NULL. */
1408 for (unsigned i = 0; i < count; i++) {
1409 assert(!mgr->real_vertex_buffer[i].is_user_buffer);
1410 mgr->real_vertex_buffer[i].buffer.resource = NULL;
1411 }
1412 } else {
1413 /* Slow path where we have to keep VBO references. */
1414 util_set_vertex_buffers(pipe, count, false, mgr->real_vertex_buffer);
1415 }
1416 mgr->vertex_buffers_dirty = false;
1417 }
1418
1419 static void
u_vbuf_split_indexed_multidraw(struct u_vbuf * mgr,struct pipe_draw_info * info,unsigned drawid_offset,unsigned * indirect_data,unsigned stride,unsigned draw_count)1420 u_vbuf_split_indexed_multidraw(struct u_vbuf *mgr, struct pipe_draw_info *info,
1421 unsigned drawid_offset,
1422 unsigned *indirect_data, unsigned stride,
1423 unsigned draw_count)
1424 {
1425 /* Increase refcount to be able to use take_index_buffer_ownership with
1426 * all draws.
1427 */
1428 if (draw_count > 1 && info->take_index_buffer_ownership)
1429 p_atomic_add(&info->index.resource->reference.count, draw_count - 1);
1430
1431 assert(info->index_size);
1432
1433 for (unsigned i = 0; i < draw_count; i++) {
1434 struct pipe_draw_start_count_bias draw;
1435 unsigned offset = i * stride / 4;
1436
1437 draw.count = indirect_data[offset + 0];
1438 info->instance_count = indirect_data[offset + 1];
1439 draw.start = indirect_data[offset + 2];
1440 draw.index_bias = indirect_data[offset + 3];
1441 info->start_instance = indirect_data[offset + 4];
1442
1443 u_vbuf_draw_vbo(mgr->pipe, info, drawid_offset, NULL, &draw, 1);
1444 }
1445 }
1446
u_vbuf_draw_vbo(struct pipe_context * pipe,const struct pipe_draw_info * info,unsigned drawid_offset,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_start_count_bias * draws,unsigned num_draws)1447 void u_vbuf_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info,
1448 unsigned drawid_offset,
1449 const struct pipe_draw_indirect_info *indirect,
1450 const struct pipe_draw_start_count_bias *draws,
1451 unsigned num_draws)
1452 {
1453 struct u_vbuf *mgr = pipe->vbuf;
1454 int start_vertex;
1455 unsigned min_index;
1456 unsigned num_vertices;
1457 bool unroll_indices = false;
1458 const uint32_t used_vb_mask = mgr->ve->used_vb_mask;
1459 uint32_t user_vb_mask = mgr->user_vb_mask & used_vb_mask;
1460 unsigned fixed_restart_index = info->index_size ? util_prim_restart_index_from_size(info->index_size) : 0;
1461
1462 uint32_t misaligned = 0;
1463 if (!mgr->caps.attrib_component_unaligned) {
1464 for (unsigned i = 0; i < ARRAY_SIZE(mgr->unaligned_vb_mask); i++) {
1465 misaligned |= mgr->ve->vb_align_mask[i] & mgr->unaligned_vb_mask[i];
1466 }
1467 }
1468 const uint32_t incompatible_vb_mask =
1469 (mgr->incompatible_vb_mask | mgr->ve->incompatible_vb_mask | misaligned) & used_vb_mask;
1470
1471 /* Normal draw. No fallback and no user buffers. */
1472 if (!incompatible_vb_mask &&
1473 !mgr->ve->incompatible_elem_mask &&
1474 !user_vb_mask &&
1475 (info->index_size != 1 || !mgr->caps.rewrite_ubyte_ibs) &&
1476 (!info->primitive_restart ||
1477 info->restart_index == fixed_restart_index ||
1478 !mgr->caps.rewrite_restart_index) &&
1479 (!info->primitive_restart || mgr->caps.supported_restart_modes & BITFIELD_BIT(info->mode)) &&
1480 mgr->caps.supported_prim_modes & BITFIELD_BIT(info->mode)) {
1481
1482 /* Set vertex buffers if needed. */
1483 if (mgr->vertex_buffers_dirty) {
1484 u_vbuf_set_driver_vertex_buffers(mgr);
1485 }
1486
1487 pipe->draw_vbo(pipe, info, drawid_offset, indirect, draws, num_draws);
1488 return;
1489 }
1490
1491 /* Increase refcount to be able to use take_index_buffer_ownership with
1492 * all draws.
1493 */
1494 if (num_draws > 1 && info->take_index_buffer_ownership)
1495 p_atomic_add(&info->index.resource->reference.count, num_draws - 1);
1496
1497 for (unsigned d = 0; d < num_draws; d++) {
1498 struct pipe_draw_info new_info = *info;
1499 struct pipe_draw_start_count_bias new_draw = draws[d];
1500
1501 /* Handle indirect (multi)draws. */
1502 if (indirect && indirect->buffer) {
1503 unsigned draw_count = 0;
1504
1505 /* num_draws can only be 1 with indirect draws. */
1506 assert(num_draws == 1);
1507
1508 /* Get the number of draws. */
1509 if (indirect->indirect_draw_count) {
1510 pipe_buffer_read(pipe, indirect->indirect_draw_count,
1511 indirect->indirect_draw_count_offset,
1512 4, &draw_count);
1513 } else {
1514 draw_count = indirect->draw_count;
1515 }
1516
1517 if (!draw_count)
1518 goto cleanup;
1519
1520 unsigned data_size = (draw_count - 1) * indirect->stride +
1521 (new_info.index_size ? 20 : 16);
1522 unsigned *data = malloc(data_size);
1523 if (!data)
1524 goto cleanup; /* report an error? */
1525
1526 /* Read the used buffer range only once, because the read can be
1527 * uncached.
1528 */
1529 pipe_buffer_read(pipe, indirect->buffer, indirect->offset, data_size,
1530 data);
1531
1532 if (info->index_size) {
1533 /* Indexed multidraw. */
1534 unsigned index_bias0 = data[3];
1535 bool index_bias_same = true;
1536
1537 /* If we invoke the translate path, we have to split the multidraw. */
1538 if (incompatible_vb_mask ||
1539 mgr->ve->incompatible_elem_mask) {
1540 u_vbuf_split_indexed_multidraw(mgr, &new_info, drawid_offset, data,
1541 indirect->stride, draw_count);
1542 free(data);
1543 /* We're done (as num_draws is 1), so return early. */
1544 return;
1545 }
1546
1547 /* See if index_bias is the same for all draws. */
1548 for (unsigned i = 1; i < draw_count; i++) {
1549 if (data[i * indirect->stride / 4 + 3] != index_bias0) {
1550 index_bias_same = false;
1551 break;
1552 }
1553 }
1554
1555 /* Split the multidraw if index_bias is different. */
1556 if (!index_bias_same) {
1557 u_vbuf_split_indexed_multidraw(mgr, &new_info, drawid_offset, data,
1558 indirect->stride, draw_count);
1559 free(data);
1560 /* We're done (as num_draws is 1), so return early. */
1561 return;
1562 }
1563
1564 /* If we don't need to use the translate path and index_bias is
1565 * the same, we can process the multidraw with the time complexity
1566 * equal to 1 draw call (except for the index range computation).
1567 * We only need to compute the index range covering all draw calls
1568 * of the multidraw.
1569 *
1570 * The driver will not look at these values because indirect != NULL.
1571 * These values determine the user buffer bounds to upload.
1572 */
1573 new_draw.index_bias = index_bias0;
1574 new_info.index_bounds_valid = true;
1575 new_info.min_index = ~0u;
1576 new_info.max_index = 0;
1577 new_info.start_instance = ~0u;
1578 unsigned end_instance = 0;
1579
1580 struct pipe_transfer *transfer = NULL;
1581 const uint8_t *indices;
1582
1583 if (info->has_user_indices) {
1584 indices = (uint8_t*)info->index.user;
1585 } else {
1586 indices = (uint8_t*)pipe_buffer_map(pipe, info->index.resource,
1587 PIPE_MAP_READ, &transfer);
1588 }
1589
1590 for (unsigned i = 0; i < draw_count; i++) {
1591 unsigned offset = i * indirect->stride / 4;
1592 unsigned start = data[offset + 2];
1593 unsigned count = data[offset + 0];
1594 unsigned start_instance = data[offset + 4];
1595 unsigned instance_count = data[offset + 1];
1596
1597 if (!count || !instance_count)
1598 continue;
1599
1600 /* Update the ranges of instances. */
1601 new_info.start_instance = MIN2(new_info.start_instance,
1602 start_instance);
1603 end_instance = MAX2(end_instance, start_instance + instance_count);
1604
1605 /* Update the index range. */
1606 unsigned min, max;
1607 u_vbuf_get_minmax_index_mapped(&new_info, count,
1608 indices +
1609 new_info.index_size * start,
1610 &min, &max);
1611
1612 new_info.min_index = MIN2(new_info.min_index, min);
1613 new_info.max_index = MAX2(new_info.max_index, max);
1614 }
1615 free(data);
1616
1617 if (transfer)
1618 pipe_buffer_unmap(pipe, transfer);
1619
1620 /* Set the final instance count. */
1621 new_info.instance_count = end_instance - new_info.start_instance;
1622
1623 if (new_info.start_instance == ~0u || !new_info.instance_count)
1624 goto cleanup;
1625 } else {
1626 /* Non-indexed multidraw.
1627 *
1628 * Keep the draw call indirect and compute minimums & maximums,
1629 * which will determine the user buffer bounds to upload, but
1630 * the driver will not look at these values because indirect != NULL.
1631 *
1632 * This efficiently processes the multidraw with the time complexity
1633 * equal to 1 draw call.
1634 */
1635 new_draw.start = ~0u;
1636 new_info.start_instance = ~0u;
1637 unsigned end_vertex = 0;
1638 unsigned end_instance = 0;
1639
1640 for (unsigned i = 0; i < draw_count; i++) {
1641 unsigned offset = i * indirect->stride / 4;
1642 unsigned start = data[offset + 2];
1643 unsigned count = data[offset + 0];
1644 unsigned start_instance = data[offset + 3];
1645 unsigned instance_count = data[offset + 1];
1646
1647 new_draw.start = MIN2(new_draw.start, start);
1648 new_info.start_instance = MIN2(new_info.start_instance,
1649 start_instance);
1650
1651 end_vertex = MAX2(end_vertex, start + count);
1652 end_instance = MAX2(end_instance, start_instance + instance_count);
1653 }
1654 free(data);
1655
1656 /* Set the final counts. */
1657 new_draw.count = end_vertex - new_draw.start;
1658 new_info.instance_count = end_instance - new_info.start_instance;
1659
1660 if (new_draw.start == ~0u || !new_draw.count || !new_info.instance_count)
1661 goto cleanup;
1662 }
1663 } else {
1664 if ((!indirect && !new_draw.count) || !new_info.instance_count)
1665 goto cleanup;
1666 }
1667
1668 if (new_info.index_size) {
1669 /* See if anything needs to be done for per-vertex attribs. */
1670 if (u_vbuf_need_minmax_index(mgr, misaligned)) {
1671 unsigned max_index;
1672
1673 if (new_info.index_bounds_valid) {
1674 min_index = new_info.min_index;
1675 max_index = new_info.max_index;
1676 } else {
1677 u_vbuf_get_minmax_index(mgr->pipe, &new_info, &new_draw,
1678 &min_index, &max_index);
1679 }
1680
1681 assert(min_index <= max_index);
1682
1683 start_vertex = min_index + new_draw.index_bias;
1684 num_vertices = max_index + 1 - min_index;
1685
1686 /* Primitive restart doesn't work when unrolling indices.
1687 * We would have to break this drawing operation into several ones. */
1688 /* Use some heuristic to see if unrolling indices improves
1689 * performance. */
1690 if (!indirect &&
1691 !new_info.primitive_restart &&
1692 util_is_vbo_upload_ratio_too_large(new_draw.count, num_vertices) &&
1693 !u_vbuf_mapping_vertex_buffer_blocks(mgr, misaligned)) {
1694 unroll_indices = true;
1695 user_vb_mask &= ~(mgr->ve->nonzero_stride_vb_mask &
1696 mgr->ve->noninstance_vb_mask_any);
1697 }
1698 } else {
1699 /* Nothing to do for per-vertex attribs. */
1700 start_vertex = 0;
1701 num_vertices = 0;
1702 min_index = 0;
1703 }
1704 } else {
1705 start_vertex = new_draw.start;
1706 num_vertices = new_draw.count;
1707 min_index = 0;
1708 }
1709
1710 /* Translate vertices with non-native layouts or formats. */
1711 if (unroll_indices ||
1712 incompatible_vb_mask ||
1713 mgr->ve->incompatible_elem_mask) {
1714 if (!u_vbuf_translate_begin(mgr, &new_info, &new_draw,
1715 start_vertex, num_vertices,
1716 min_index, unroll_indices, misaligned)) {
1717 debug_warn_once("u_vbuf_translate_begin() failed");
1718 goto cleanup;
1719 }
1720
1721 if (unroll_indices) {
1722 if (!new_info.has_user_indices && info->take_index_buffer_ownership)
1723 pipe_drop_resource_references(new_info.index.resource, 1);
1724 new_info.index_size = 0;
1725 new_draw.index_bias = 0;
1726 new_info.index_bounds_valid = true;
1727 new_info.min_index = 0;
1728 new_info.max_index = new_draw.count - 1;
1729 new_draw.start = 0;
1730 }
1731
1732 user_vb_mask &= ~(incompatible_vb_mask |
1733 mgr->ve->incompatible_vb_mask_all);
1734 mgr->vertex_buffers_dirty = true;
1735 }
1736
1737 /* Upload user buffers. */
1738 if (user_vb_mask) {
1739 if (u_vbuf_upload_buffers(mgr, start_vertex, num_vertices,
1740 new_info.start_instance,
1741 new_info.instance_count) != PIPE_OK) {
1742 debug_warn_once("u_vbuf_upload_buffers() failed");
1743 goto cleanup;
1744 }
1745
1746 mgr->vertex_buffers_dirty = true;
1747 }
1748
1749 /*
1750 if (unroll_indices) {
1751 printf("unrolling indices: start_vertex = %i, num_vertices = %i\n",
1752 start_vertex, num_vertices);
1753 util_dump_draw_info(stdout, info);
1754 printf("\n");
1755 }
1756
1757 unsigned i;
1758 for (i = 0; i < mgr->nr_vertex_buffers; i++) {
1759 printf("input %i: ", i);
1760 util_dump_vertex_buffer(stdout, mgr->vertex_buffer+i);
1761 printf("\n");
1762 }
1763 for (i = 0; i < mgr->nr_real_vertex_buffers; i++) {
1764 printf("real %i: ", i);
1765 util_dump_vertex_buffer(stdout, mgr->real_vertex_buffer+i);
1766 printf("\n");
1767 }
1768 */
1769
1770 u_upload_unmap(pipe->stream_uploader);
1771 if (mgr->vertex_buffers_dirty)
1772 u_vbuf_set_driver_vertex_buffers(mgr);
1773
1774 if ((new_info.index_size == 1 && mgr->caps.rewrite_ubyte_ibs) ||
1775 (new_info.primitive_restart &&
1776 ((new_info.restart_index != fixed_restart_index && mgr->caps.rewrite_restart_index) ||
1777 !(mgr->caps.supported_restart_modes & BITFIELD_BIT(new_info.mode)))) ||
1778 !(mgr->caps.supported_prim_modes & BITFIELD_BIT(new_info.mode))) {
1779 util_primconvert_save_flatshade_first(mgr->pc, mgr->flatshade_first);
1780 util_primconvert_draw_vbo(mgr->pc, &new_info, drawid_offset, indirect, &new_draw, 1);
1781 } else
1782 pipe->draw_vbo(pipe, &new_info, drawid_offset, indirect, &new_draw, 1);
1783 if (info->increment_draw_id)
1784 drawid_offset++;
1785 }
1786
1787 if (mgr->using_translate) {
1788 u_vbuf_translate_end(mgr);
1789 }
1790 return;
1791
1792 cleanup:
1793 if (info->take_index_buffer_ownership) {
1794 struct pipe_resource *indexbuf = info->index.resource;
1795 pipe_resource_reference(&indexbuf, NULL);
1796 }
1797 }
1798
u_vbuf_save_vertex_elements(struct u_vbuf * mgr)1799 void u_vbuf_save_vertex_elements(struct u_vbuf *mgr)
1800 {
1801 assert(!mgr->ve_saved);
1802 mgr->ve_saved = mgr->ve;
1803 }
1804
u_vbuf_restore_vertex_elements(struct u_vbuf * mgr)1805 void u_vbuf_restore_vertex_elements(struct u_vbuf *mgr)
1806 {
1807 if (mgr->ve != mgr->ve_saved) {
1808 struct pipe_context *pipe = mgr->pipe;
1809
1810 mgr->ve = mgr->ve_saved;
1811 pipe->bind_vertex_elements_state(pipe,
1812 mgr->ve ? mgr->ve->driver_cso : NULL);
1813 }
1814 mgr->ve_saved = NULL;
1815 }
1816