1 /*
2 * Copyright (c) 2008-2024 Broadcom. All Rights Reserved.
3 * The term “Broadcom” refers to Broadcom Inc.
4 * and/or its subsidiaries.
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "svga_cmd.h"
9
10 #include "pipe/p_state.h"
11 #include "pipe/p_defines.h"
12 #include "util/u_inlines.h"
13 #include "util/u_thread.h"
14 #include "util/u_math.h"
15 #include "util/u_memory.h"
16 #include "util/u_resource.h"
17
18 #include "svga_context.h"
19 #include "svga_screen.h"
20 #include "svga_resource_buffer.h"
21 #include "svga_resource_buffer_upload.h"
22 #include "svga_resource_texture.h"
23 #include "svga_sampler_view.h"
24 #include "svga_winsys.h"
25 #include "svga_debug.h"
26
27
28 /**
29 * Determine what buffers eventually need hardware backing.
30 *
31 * Vertex- and index buffers need hardware backing. Constant buffers
32 * do on vgpu10. Staging texture-upload buffers do when they are
33 * supported.
34 */
35 static inline bool
svga_buffer_needs_hw_storage(const struct svga_screen * ss,const struct pipe_resource * template)36 svga_buffer_needs_hw_storage(const struct svga_screen *ss,
37 const struct pipe_resource *template)
38 {
39 unsigned bind_mask = (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER |
40 PIPE_BIND_SAMPLER_VIEW | PIPE_BIND_STREAM_OUTPUT |
41 PIPE_BIND_SHADER_BUFFER | PIPE_BIND_COMMAND_ARGS_BUFFER);
42
43 if (ss->sws->have_vgpu10) {
44 /*
45 * Driver-created upload const0- and staging texture upload buffers
46 * tagged with PIPE_BIND_CUSTOM
47 */
48 bind_mask |= PIPE_BIND_CUSTOM;
49 /**
50 * Uniform buffer objects.
51 * Don't create hardware storage for state-tracker constant buffers,
52 * because we frequently map them for reading and writing, and
53 * the length of those buffers are always small, so it is better
54 * to just use system memory.
55 */
56 }
57
58 if (template->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT)
59 return true;
60
61 return !!(template->bind & bind_mask);
62 }
63
64
65 static inline bool
need_buf_readback(struct svga_context * svga,struct pipe_transfer * st)66 need_buf_readback(struct svga_context *svga,
67 struct pipe_transfer *st)
68 {
69 struct svga_buffer *sbuf = svga_buffer(st->resource);
70
71 if (st->usage != PIPE_MAP_READ)
72 return false;
73
74 /* No buffer surface has been created */
75 if (!sbuf->bufsurf)
76 return false;
77
78 return ((sbuf->dirty ||
79 sbuf->bufsurf->surface_state == SVGA_SURFACE_STATE_RENDERED) &&
80 !sbuf->key.coherent && !svga->swc->force_coherent);
81 }
82
83
84 /**
85 * Create a buffer transfer.
86 *
87 * Unlike texture DMAs (which are written immediately to the command buffer and
88 * therefore inherently serialized with other context operations), for buffers
89 * we try to coalesce multiple range mappings (i.e, multiple calls to this
90 * function) into a single DMA command, for better efficiency in command
91 * processing. This means we need to exercise extra care here to ensure that
92 * the end result is exactly the same as if one DMA was used for every mapped
93 * range.
94 */
95 void *
svga_buffer_transfer_map(struct pipe_context * pipe,struct pipe_resource * resource,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** ptransfer)96 svga_buffer_transfer_map(struct pipe_context *pipe,
97 struct pipe_resource *resource,
98 unsigned level,
99 unsigned usage,
100 const struct pipe_box *box,
101 struct pipe_transfer **ptransfer)
102 {
103 struct svga_context *svga = svga_context(pipe);
104 struct svga_screen *ss = svga_screen(pipe->screen);
105 struct svga_buffer *sbuf = svga_buffer(resource);
106 struct pipe_transfer *transfer;
107 uint8_t *map = NULL;
108 int64_t begin = svga_get_time(svga);
109
110 SVGA_STATS_TIME_PUSH(svga_sws(svga), SVGA_STATS_TIME_BUFFERTRANSFERMAP);
111
112 assert(box->y == 0);
113 assert(box->z == 0);
114 assert(box->height == 1);
115 assert(box->depth == 1);
116
117 transfer = MALLOC_STRUCT(pipe_transfer);
118 if (!transfer) {
119 goto done;
120 }
121
122 transfer->resource = resource;
123 transfer->level = level;
124 transfer->usage = usage;
125 transfer->box = *box;
126 transfer->stride = 0;
127 transfer->layer_stride = 0;
128
129 if (usage & PIPE_MAP_WRITE) {
130 /* If we write to the buffer for any reason, free any saved translated
131 * vertices.
132 */
133 pipe_resource_reference(&sbuf->translated_indices.buffer, NULL);
134 }
135
136 /* If it is a read transfer and the buffer is dirty or the buffer is bound
137 * to a uav, we will need to read the subresource content from the device.
138 */
139 if (need_buf_readback(svga, transfer)) {
140 /* Host-side buffers can be dirtied with vgpu10 features
141 * (streamout and buffer copy) and sm5 feature via uav.
142 */
143 assert(svga_have_vgpu10(svga));
144
145 if (!sbuf->user) {
146 (void) svga_buffer_handle(svga, resource, sbuf->bind_flags);
147 }
148
149 if (sbuf->dma.pending) {
150 svga_buffer_upload_flush(svga, sbuf);
151 svga_context_finish(svga);
152 }
153
154 assert(sbuf->handle);
155
156 SVGA_RETRY(svga, SVGA3D_ReadbackGBSurface(svga->swc, sbuf->handle));
157 svga->hud.num_readbacks++;
158
159 svga_context_finish(svga);
160
161 sbuf->dirty = false;
162
163 /* Mark the buffer surface state as UPDATED */
164 assert(sbuf->bufsurf);
165 sbuf->bufsurf->surface_state = SVGA_SURFACE_STATE_UPDATED;
166 }
167
168 if (usage & PIPE_MAP_WRITE) {
169 if ((usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) &&
170 !(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT)) {
171 /*
172 * Flush any pending primitives, finish writing any pending DMA
173 * commands, and tell the host to discard the buffer contents on
174 * the next DMA operation.
175 */
176
177 svga_hwtnl_flush_buffer(svga, resource);
178
179 if (sbuf->dma.pending) {
180 svga_buffer_upload_flush(svga, sbuf);
181
182 /*
183 * Instead of flushing the context command buffer, simply discard
184 * the current hwbuf, and start a new one.
185 * With GB objects, the map operation takes care of this
186 * if passed the PIPE_MAP_DISCARD_WHOLE_RESOURCE flag,
187 * and the old backing store is busy.
188 */
189
190 if (!svga_have_gb_objects(svga))
191 svga_buffer_destroy_hw_storage(ss, sbuf);
192 }
193
194 sbuf->map.num_ranges = 0;
195 sbuf->dma.flags.discard = true;
196 }
197
198 if (usage & PIPE_MAP_UNSYNCHRONIZED) {
199 if (!sbuf->map.num_ranges) {
200 /*
201 * No pending ranges to upload so far, so we can tell the host to
202 * not synchronize on the next DMA command.
203 */
204
205 sbuf->dma.flags.unsynchronized = true;
206 }
207 } else {
208 /*
209 * Synchronizing, so flush any pending primitives, finish writing any
210 * pending DMA command, and ensure the next DMA will be done in order.
211 */
212
213 svga_hwtnl_flush_buffer(svga, resource);
214
215 if (sbuf->dma.pending) {
216 svga_buffer_upload_flush(svga, sbuf);
217
218 if (svga_buffer_has_hw_storage(sbuf)) {
219 /*
220 * We have a pending DMA upload from a hardware buffer, therefore
221 * we need to ensure that the host finishes processing that DMA
222 * command before the gallium frontend can start overwriting the
223 * hardware buffer.
224 *
225 * XXX: This could be avoided by tying the hardware buffer to
226 * the transfer (just as done with textures), which would allow
227 * overlapping DMAs commands to be queued on the same context
228 * buffer. However, due to the likelihood of software vertex
229 * processing, it is more convenient to hold on to the hardware
230 * buffer, allowing to quickly access the contents from the CPU
231 * without having to do a DMA download from the host.
232 */
233
234 if (usage & PIPE_MAP_DONTBLOCK) {
235 /*
236 * Flushing the command buffer here will most likely cause
237 * the map of the hwbuf below to block, so preemptively
238 * return NULL here if DONTBLOCK is set to prevent unnecessary
239 * command buffer flushes.
240 */
241
242 FREE(transfer);
243 goto done;
244 }
245
246 svga_context_flush(svga, NULL);
247 }
248 }
249
250 sbuf->dma.flags.unsynchronized = false;
251 }
252 }
253
254 if (!sbuf->swbuf && !svga_buffer_has_hw_storage(sbuf)) {
255 if (svga_buffer_create_hw_storage(ss, sbuf, sbuf->bind_flags) != PIPE_OK) {
256 /*
257 * We can't create a hardware buffer big enough, so create a malloc
258 * buffer instead.
259 */
260 if (0) {
261 debug_printf("%s: failed to allocate %u KB of DMA, "
262 "splitting DMA transfers\n",
263 __func__,
264 (sbuf->b.width0 + 1023)/1024);
265 }
266
267 sbuf->swbuf = align_malloc(sbuf->b.width0, 16);
268 if (!sbuf->swbuf) {
269 FREE(transfer);
270 goto done;
271 }
272 }
273 }
274
275 if (sbuf->swbuf) {
276 /* User/malloc buffer */
277 map = sbuf->swbuf;
278 }
279 else if (svga_buffer_has_hw_storage(sbuf)) {
280 bool retry;
281
282 map = SVGA_TRY_MAP(svga_buffer_hw_storage_map
283 (svga, sbuf, transfer->usage, &retry), retry);
284 if (map == NULL && retry) {
285 /*
286 * At this point, svga_buffer_get_transfer() has already
287 * hit the DISCARD_WHOLE_RESOURCE path and flushed HWTNL
288 * for this buffer.
289 */
290 svga_retry_enter(svga);
291 svga_context_flush(svga, NULL);
292 map = svga_buffer_hw_storage_map(svga, sbuf, transfer->usage, &retry);
293 svga_retry_exit(svga);
294 }
295 }
296 else {
297 map = NULL;
298 }
299
300 if (map) {
301 ++sbuf->map.count;
302 map += transfer->box.x;
303 *ptransfer = transfer;
304 } else {
305 FREE(transfer);
306 }
307
308 svga->hud.map_buffer_time += (svga_get_time(svga) - begin);
309
310 done:
311 SVGA_STATS_TIME_POP(svga_sws(svga));
312 return map;
313 }
314
315
316 void
svga_buffer_transfer_flush_region(struct pipe_context * pipe,struct pipe_transfer * transfer,const struct pipe_box * box)317 svga_buffer_transfer_flush_region(struct pipe_context *pipe,
318 struct pipe_transfer *transfer,
319 const struct pipe_box *box)
320 {
321 struct svga_screen *ss = svga_screen(pipe->screen);
322 struct svga_buffer *sbuf = svga_buffer(transfer->resource);
323 struct svga_context *svga = svga_context(pipe);
324 unsigned offset = transfer->box.x + box->x;
325 unsigned length = box->width;
326
327 assert(transfer->usage & PIPE_MAP_WRITE);
328 assert(transfer->usage & PIPE_MAP_FLUSH_EXPLICIT);
329
330 if (!(svga->swc->force_coherent || sbuf->key.coherent) || sbuf->swbuf) {
331 mtx_lock(&ss->swc_mutex);
332 svga_buffer_add_range(sbuf, offset, offset + length);
333 mtx_unlock(&ss->swc_mutex);
334 }
335 }
336
337
338 void
svga_buffer_transfer_unmap(struct pipe_context * pipe,struct pipe_transfer * transfer)339 svga_buffer_transfer_unmap(struct pipe_context *pipe,
340 struct pipe_transfer *transfer)
341 {
342 struct svga_screen *ss = svga_screen(pipe->screen);
343 struct svga_context *svga = svga_context(pipe);
344 struct svga_buffer *sbuf = svga_buffer(transfer->resource);
345
346 SVGA_STATS_TIME_PUSH(svga_sws(svga), SVGA_STATS_TIME_BUFFERTRANSFERUNMAP);
347
348 mtx_lock(&ss->swc_mutex);
349
350 assert(sbuf->map.count);
351 if (sbuf->map.count) {
352 --sbuf->map.count;
353 }
354
355 if (svga_buffer_has_hw_storage(sbuf)) {
356
357 /* Note: we may wind up flushing here and unmapping other buffers
358 * which leads to recursively locking ss->swc_mutex.
359 */
360 svga_buffer_hw_storage_unmap(svga, sbuf);
361 }
362
363 if (transfer->usage & PIPE_MAP_WRITE) {
364 if (!(transfer->usage & PIPE_MAP_FLUSH_EXPLICIT)) {
365 /*
366 * Mapped range not flushed explicitly, so flush the whole buffer,
367 * and tell the host to discard the contents when processing the DMA
368 * command.
369 */
370
371 SVGA_DBG(DEBUG_DMA, "flushing the whole buffer\n");
372
373 sbuf->dma.flags.discard = true;
374
375 if (!(svga->swc->force_coherent || sbuf->key.coherent) || sbuf->swbuf)
376 svga_buffer_add_range(sbuf, 0, sbuf->b.width0);
377 }
378
379 if (sbuf->swbuf &&
380 (!sbuf->bind_flags || (sbuf->bind_flags & PIPE_BIND_CONSTANT_BUFFER))) {
381 /*
382 * Since the constant buffer is in system buffer, we need
383 * to set the constant buffer dirty bits, so that the context
384 * can update the changes in the device.
385 * According to the GL spec, buffer bound to other contexts will
386 * have to be explicitly rebound by the user to have the changes take
387 * into effect.
388 */
389 svga->dirty |= SVGA_NEW_CONST_BUFFER;
390 }
391 }
392
393 mtx_unlock(&ss->swc_mutex);
394 FREE(transfer);
395 SVGA_STATS_TIME_POP(svga_sws(svga));
396 }
397
398
399 void
svga_resource_destroy(struct pipe_screen * screen,struct pipe_resource * buf)400 svga_resource_destroy(struct pipe_screen *screen,
401 struct pipe_resource *buf)
402 {
403 if (buf->target == PIPE_BUFFER) {
404 struct svga_screen *ss = svga_screen(screen);
405 struct svga_buffer *sbuf = svga_buffer(buf);
406
407 assert(!p_atomic_read(&buf->reference.count));
408
409 assert(!sbuf->dma.pending);
410
411 if (sbuf->handle)
412 svga_buffer_destroy_host_surface(ss, sbuf);
413
414 if (sbuf->uploaded.buffer)
415 pipe_resource_reference(&sbuf->uploaded.buffer, NULL);
416
417 if (sbuf->hwbuf)
418 svga_buffer_destroy_hw_storage(ss, sbuf);
419
420 if (sbuf->swbuf && !sbuf->user)
421 align_free(sbuf->swbuf);
422
423 pipe_resource_reference(&sbuf->translated_indices.buffer, NULL);
424
425 ss->hud.total_resource_bytes -= sbuf->size;
426 assert(ss->hud.num_resources > 0);
427 if (ss->hud.num_resources > 0)
428 ss->hud.num_resources--;
429
430 FREE(sbuf);
431 } else {
432 struct svga_screen *ss = svga_screen(screen);
433 struct svga_texture *tex = svga_texture(buf);
434
435 ss->texture_timestamp++;
436
437 svga_sampler_view_reference(&tex->cached_view, NULL);
438
439 /*
440 DBG("%s deleting %p\n", __func__, (void *) tex);
441 */
442 SVGA_DBG(DEBUG_DMA, "unref sid %p (texture)\n", tex->handle);
443
444 bool to_invalidate = svga_was_texture_rendered_to(tex);
445 svga_screen_surface_destroy(ss, &tex->key, to_invalidate, &tex->handle);
446
447 /* Destroy the backed surface handle if exists */
448 if (tex->backed_handle)
449 svga_screen_surface_destroy(ss, &tex->backed_key, to_invalidate, &tex->backed_handle);
450
451 ss->hud.total_resource_bytes -= tex->size;
452
453 FREE(tex->defined);
454 FREE(tex->rendered_to);
455 FREE(tex->dirty);
456 FREE(tex);
457
458 assert(ss->hud.num_resources > 0);
459 if (ss->hud.num_resources > 0)
460 ss->hud.num_resources--;
461 }
462 }
463
464 struct pipe_resource *
svga_buffer_create(struct pipe_screen * screen,const struct pipe_resource * template)465 svga_buffer_create(struct pipe_screen *screen,
466 const struct pipe_resource *template)
467 {
468 struct svga_screen *ss = svga_screen(screen);
469 struct svga_buffer *sbuf;
470 unsigned bind_flags;
471
472 SVGA_STATS_TIME_PUSH(ss->sws, SVGA_STATS_TIME_CREATEBUFFER);
473
474 sbuf = CALLOC_STRUCT(svga_buffer);
475 if (!sbuf)
476 goto error1;
477
478 sbuf->b = *template;
479 pipe_reference_init(&sbuf->b.reference, 1);
480 sbuf->b.screen = screen;
481 bind_flags = template->bind & ~PIPE_BIND_CUSTOM;
482
483 list_inithead(&sbuf->surfaces);
484
485 if (bind_flags & PIPE_BIND_CONSTANT_BUFFER) {
486 /* Constant buffers can only have the PIPE_BIND_CONSTANT_BUFFER
487 * flag set.
488 */
489 if (ss->sws->have_vgpu10) {
490 bind_flags = PIPE_BIND_CONSTANT_BUFFER;
491 }
492 }
493
494 /* Although svga device only requires constant buffer size to be
495 * in multiples of 16, in order to allow bind_flags promotion,
496 * we are mandating all buffer size to be in multiples of 16.
497 */
498 sbuf->b.width0 = align(sbuf->b.width0, 16);
499
500 if (svga_buffer_needs_hw_storage(ss, template)) {
501
502 /* If the buffer is not used for constant buffer, set
503 * the vertex/index bind flags as well so that the buffer will be
504 * accepted for those uses.
505 * Note that the PIPE_BIND_ flags we get from the gallium frontend are
506 * just a hint about how the buffer may be used. And OpenGL buffer
507 * object may be used for many different things.
508 * Also note that we do not unconditionally set the streamout
509 * bind flag since streamout buffer is an output buffer and
510 * might have performance implication.
511 */
512 if (!(template->bind & PIPE_BIND_CONSTANT_BUFFER) &&
513 !(template->bind & PIPE_BIND_CUSTOM)) {
514 /* Not a constant- or staging buffer.
515 * The buffer may be used for vertex data or indexes.
516 */
517 bind_flags |= (PIPE_BIND_VERTEX_BUFFER |
518 PIPE_BIND_INDEX_BUFFER);
519
520 /* It may be used for shader resource as well. */
521 bind_flags |= PIPE_BIND_SAMPLER_VIEW;
522 }
523
524 if (svga_buffer_create_host_surface(ss, sbuf, bind_flags) != PIPE_OK)
525 goto error2;
526 }
527 else {
528 sbuf->swbuf = align_malloc(sbuf->b.width0, 64);
529 if (!sbuf->swbuf)
530 goto error2;
531
532 /* Since constant buffer is usually small, it is much cheaper to
533 * use system memory for the data just as it is being done for
534 * the default constant buffer.
535 */
536 if ((bind_flags & PIPE_BIND_CONSTANT_BUFFER) || !bind_flags)
537 sbuf->use_swbuf = true;
538 }
539
540 debug_reference(&sbuf->b.reference,
541 (debug_reference_descriptor)debug_describe_resource, 0);
542
543 sbuf->bind_flags = bind_flags;
544 sbuf->size = util_resource_size(&sbuf->b);
545 ss->hud.total_resource_bytes += sbuf->size;
546
547 ss->hud.num_resources++;
548 SVGA_STATS_TIME_POP(ss->sws);
549
550 return &sbuf->b;
551
552 error2:
553 FREE(sbuf);
554 error1:
555 SVGA_STATS_TIME_POP(ss->sws);
556 return NULL;
557 }
558
559
560 struct pipe_resource *
svga_user_buffer_create(struct pipe_screen * screen,void * ptr,unsigned bytes,unsigned bind)561 svga_user_buffer_create(struct pipe_screen *screen,
562 void *ptr,
563 unsigned bytes,
564 unsigned bind)
565 {
566 struct svga_buffer *sbuf;
567 struct svga_screen *ss = svga_screen(screen);
568
569 sbuf = CALLOC_STRUCT(svga_buffer);
570 if (!sbuf)
571 goto no_sbuf;
572
573 pipe_reference_init(&sbuf->b.reference, 1);
574 sbuf->b.screen = screen;
575 sbuf->b.format = PIPE_FORMAT_R8_UNORM; /* ?? */
576 sbuf->b.usage = PIPE_USAGE_IMMUTABLE;
577 sbuf->b.bind = bind;
578 sbuf->b.width0 = bytes;
579 sbuf->b.height0 = 1;
580 sbuf->b.depth0 = 1;
581 sbuf->b.array_size = 1;
582
583 sbuf->bind_flags = bind;
584 sbuf->swbuf = ptr;
585 sbuf->user = true;
586
587 debug_reference(&sbuf->b.reference,
588 (debug_reference_descriptor)debug_describe_resource, 0);
589
590 ss->hud.num_resources++;
591
592 return &sbuf->b;
593
594 no_sbuf:
595 return NULL;
596 }
597