1 /*
2 * Authors:
3 * Adam Rak <[email protected]>
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "pipe/p_defines.h"
8 #include "pipe/p_state.h"
9 #include "pipe/p_context.h"
10 #include "util/u_blitter.h"
11 #include "util/list.h"
12 #include "util/u_transfer.h"
13 #include "util/u_surface.h"
14 #include "util/u_pack_color.h"
15 #include "util/u_math.h"
16 #include "util/u_memory.h"
17 #include "util/u_inlines.h"
18 #include "util/u_framebuffer.h"
19 #include "r600_shader.h"
20 #include "r600_pipe.h"
21 #include "r600_formats.h"
22 #include "compute_memory_pool.h"
23 #include "evergreen_compute.h"
24 #include "evergreen_compute_internal.h"
25 #include <inttypes.h>
26
27 #define ITEM_ALIGNMENT 1024
28
29 /* A few forward declarations of static functions */
30 static void compute_memory_shadow(struct compute_memory_pool* pool,
31 struct pipe_context *pipe, int device_to_host);
32
33 static void compute_memory_defrag(struct compute_memory_pool *pool,
34 struct pipe_resource *src, struct pipe_resource *dst,
35 struct pipe_context *pipe);
36
37 static int compute_memory_promote_item(struct compute_memory_pool *pool,
38 struct compute_memory_item *item, struct pipe_context *pipe,
39 int64_t allocated);
40
41 static void compute_memory_move_item(struct compute_memory_pool *pool,
42 struct pipe_resource *src, struct pipe_resource *dst,
43 struct compute_memory_item *item, uint64_t new_start_in_dw,
44 struct pipe_context *pipe);
45
46 static void compute_memory_transfer(struct compute_memory_pool* pool,
47 struct pipe_context * pipe, int device_to_host,
48 struct compute_memory_item* chunk, void* data,
49 int offset_in_chunk, int size);
50
51 /**
52 * Creates a new pool.
53 */
compute_memory_pool_new(struct r600_screen * rscreen)54 struct compute_memory_pool* compute_memory_pool_new(
55 struct r600_screen * rscreen)
56 {
57 struct compute_memory_pool* pool = (struct compute_memory_pool*)
58 CALLOC(sizeof(struct compute_memory_pool), 1);
59 if (!pool)
60 return NULL;
61
62 COMPUTE_DBG(rscreen, "* compute_memory_pool_new()\n");
63
64 pool->screen = rscreen;
65 pool->item_list = (struct list_head *)
66 CALLOC(sizeof(struct list_head), 1);
67 pool->unallocated_list = (struct list_head *)
68 CALLOC(sizeof(struct list_head), 1);
69 list_inithead(pool->item_list);
70 list_inithead(pool->unallocated_list);
71 return pool;
72 }
73
74 /**
75 * Initializes the pool with a size of \a initial_size_in_dw.
76 * \param pool The pool to be initialized.
77 * \param initial_size_in_dw The initial size.
78 * \see compute_memory_grow_defrag_pool
79 */
compute_memory_pool_init(struct compute_memory_pool * pool,unsigned initial_size_in_dw)80 static void compute_memory_pool_init(struct compute_memory_pool * pool,
81 unsigned initial_size_in_dw)
82 {
83
84 COMPUTE_DBG(pool->screen, "* compute_memory_pool_init() initial_size_in_dw = %u\n",
85 initial_size_in_dw);
86
87 pool->size_in_dw = initial_size_in_dw;
88 pool->bo = r600_compute_buffer_alloc_vram(pool->screen,
89 pool->size_in_dw * 4);
90 }
91
92 /**
93 * Frees all stuff in the pool and the pool struct itself too.
94 */
compute_memory_pool_delete(struct compute_memory_pool * pool)95 void compute_memory_pool_delete(struct compute_memory_pool* pool)
96 {
97 COMPUTE_DBG(pool->screen, "* compute_memory_pool_delete()\n");
98 free(pool->shadow);
99 r600_resource_reference(&pool->bo, NULL);
100 /* In theory, all of the items were freed in compute_memory_free.
101 * Just delete the list heads
102 */
103 free(pool->item_list);
104 free(pool->unallocated_list);
105 /* And then the pool itself */
106 free(pool);
107 }
108
109 /**
110 * Reallocates and defragments the pool, conserves data.
111 * \returns -1 if it fails, 0 otherwise
112 * \see compute_memory_finalize_pending
113 */
compute_memory_grow_defrag_pool(struct compute_memory_pool * pool,struct pipe_context * pipe,int new_size_in_dw)114 static int compute_memory_grow_defrag_pool(struct compute_memory_pool *pool,
115 struct pipe_context *pipe, int new_size_in_dw)
116 {
117 new_size_in_dw = align(new_size_in_dw, ITEM_ALIGNMENT);
118
119 COMPUTE_DBG(pool->screen, "* compute_memory_grow_defrag_pool() "
120 "new_size_in_dw = %d (%d bytes)\n",
121 new_size_in_dw, new_size_in_dw * 4);
122
123 assert(new_size_in_dw >= pool->size_in_dw);
124
125 if (!pool->bo) {
126 compute_memory_pool_init(pool, MAX2(new_size_in_dw, 1024 * 16));
127 } else {
128 struct r600_resource *temp = NULL;
129
130 temp = r600_compute_buffer_alloc_vram(pool->screen, new_size_in_dw * 4);
131
132 if (temp != NULL) {
133 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
134 struct pipe_resource *dst = (struct pipe_resource *)temp;
135
136 COMPUTE_DBG(pool->screen, " Growing and defragmenting the pool "
137 "using a temporary resource\n");
138
139 compute_memory_defrag(pool, src, dst, pipe);
140
141 /* Release the old buffer */
142 r600_resource_reference(&pool->bo, NULL);
143 pool->bo = temp;
144 pool->size_in_dw = new_size_in_dw;
145 }
146 else {
147 COMPUTE_DBG(pool->screen, " The creation of the temporary resource failed\n"
148 " Falling back to using 'shadow'\n");
149
150 compute_memory_shadow(pool, pipe, 1);
151 pool->shadow = realloc(pool->shadow, new_size_in_dw * 4);
152 if (pool->shadow == NULL)
153 return -1;
154
155 pool->size_in_dw = new_size_in_dw;
156 /* Release the old buffer */
157 r600_resource_reference(&pool->bo, NULL);
158 pool->bo = r600_compute_buffer_alloc_vram(pool->screen, pool->size_in_dw * 4);
159 compute_memory_shadow(pool, pipe, 0);
160
161 if (pool->status & POOL_FRAGMENTED) {
162 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
163 compute_memory_defrag(pool, src, src, pipe);
164 }
165 }
166 }
167
168 return 0;
169 }
170
171 /**
172 * Copy pool from device to host, or host to device.
173 * \param device_to_host 1 for device->host, 0 for host->device
174 * \see compute_memory_grow_defrag_pool
175 */
compute_memory_shadow(struct compute_memory_pool * pool,struct pipe_context * pipe,int device_to_host)176 static void compute_memory_shadow(struct compute_memory_pool* pool,
177 struct pipe_context * pipe, int device_to_host)
178 {
179 struct compute_memory_item chunk;
180
181 COMPUTE_DBG(pool->screen, "* compute_memory_shadow() device_to_host = %d\n",
182 device_to_host);
183
184 chunk.id = 0;
185 chunk.start_in_dw = 0;
186 chunk.size_in_dw = pool->size_in_dw;
187 compute_memory_transfer(pool, pipe, device_to_host, &chunk,
188 pool->shadow, 0, pool->size_in_dw*4);
189 }
190
191 /**
192 * Moves all the items marked for promotion from the \a unallocated_list
193 * to the \a item_list.
194 * \return -1 if it fails, 0 otherwise
195 * \see evergreen_set_global_binding
196 */
compute_memory_finalize_pending(struct compute_memory_pool * pool,struct pipe_context * pipe)197 int compute_memory_finalize_pending(struct compute_memory_pool* pool,
198 struct pipe_context * pipe)
199 {
200 struct compute_memory_item *item, *next;
201
202 int64_t allocated = 0;
203 int64_t unallocated = 0;
204 int64_t last_pos;
205
206 int err = 0;
207
208 COMPUTE_DBG(pool->screen, "* compute_memory_finalize_pending()\n");
209
210 LIST_FOR_EACH_ENTRY(item, pool->item_list, link) {
211 COMPUTE_DBG(pool->screen, " + list: offset = %"PRIi64" id = %"PRIi64" size = %"PRIi64" "
212 "(%"PRIi64" bytes)\n", item->start_in_dw, item->id,
213 item->size_in_dw, item->size_in_dw * 4);
214 }
215
216 /* Calculate the total allocated size */
217 LIST_FOR_EACH_ENTRY(item, pool->item_list, link) {
218 allocated += align(item->size_in_dw, ITEM_ALIGNMENT);
219 }
220
221 /* Calculate the total unallocated size of the items that
222 * will be promoted to the pool */
223 LIST_FOR_EACH_ENTRY(item, pool->unallocated_list, link) {
224 if (item->status & ITEM_FOR_PROMOTING)
225 unallocated += align(item->size_in_dw, ITEM_ALIGNMENT);
226 }
227
228 if (unallocated == 0) {
229 return 0;
230 }
231
232 if (pool->size_in_dw < allocated + unallocated) {
233 err = compute_memory_grow_defrag_pool(pool, pipe, allocated + unallocated);
234 if (err == -1)
235 return -1;
236 }
237 else if (pool->status & POOL_FRAGMENTED) {
238 /* Loop through all unallocated items marked for promoting to
239 * insert them into an appropriate existing hole prior to defrag. */
240 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) {
241 if (!(item->status & ITEM_FOR_PROMOTING))
242 continue;
243
244 int64_t hole_start = 0, hole_size = 0;
245 int64_t item_size = align(item->size_in_dw, ITEM_ALIGNMENT);
246 struct compute_memory_item *alloc_item, *alloc_next;
247 LIST_FOR_EACH_ENTRY_SAFE(alloc_item, alloc_next, pool->item_list, link) {
248 if (alloc_item->start_in_dw == hole_start) {
249 hole_start += align(alloc_item->size_in_dw, ITEM_ALIGNMENT);
250 hole_size = 0;
251 } else if (alloc_item->start_in_dw > hole_start) {
252 hole_size = alloc_item->start_in_dw - hole_start;
253 }
254 }
255
256 /* Space after all items is also a hole. */
257 if (hole_size == 0 && hole_start < pool->size_in_dw)
258 hole_size = pool->size_in_dw - hole_start;
259
260 if (hole_size >= item_size) {
261 if (compute_memory_promote_item(pool, item, pipe, hole_start) != -1) {
262 item->status &= ~ITEM_FOR_PROMOTING;
263 unallocated -= item_size;
264 allocated += item_size;
265 }
266 }
267 }
268
269 if (allocated == pool->size_in_dw)
270 pool->status &= ~POOL_FRAGMENTED;
271
272 if (unallocated == 0)
273 return 0;
274
275 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
276 compute_memory_defrag(pool, src, src, pipe);
277 }
278
279 /* After defragmenting the pool, allocated is equal to the first available
280 * position for new items in the pool */
281 last_pos = allocated;
282
283 /* Loop through all the unallocated items, check if they are marked
284 * for promoting, allocate space for them and add them to the item_list. */
285 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) {
286 if (item->status & ITEM_FOR_PROMOTING) {
287 err = compute_memory_promote_item(pool, item, pipe, last_pos);
288 item->status &= ~ITEM_FOR_PROMOTING;
289
290 last_pos += align(item->size_in_dw, ITEM_ALIGNMENT);
291
292 if (err == -1)
293 return -1;
294 }
295 }
296
297 return 0;
298 }
299
300 /**
301 * Defragments the pool, so that there's no gap between items.
302 * \param pool The pool to be defragmented
303 * \param src The origin resource
304 * \param dst The destination resource
305 * \see compute_memory_grow_defrag_pool and compute_memory_finalize_pending
306 */
compute_memory_defrag(struct compute_memory_pool * pool,struct pipe_resource * src,struct pipe_resource * dst,struct pipe_context * pipe)307 static void compute_memory_defrag(struct compute_memory_pool *pool,
308 struct pipe_resource *src, struct pipe_resource *dst,
309 struct pipe_context *pipe)
310 {
311 struct compute_memory_item *item;
312 int64_t last_pos;
313
314 COMPUTE_DBG(pool->screen, "* compute_memory_defrag()\n");
315
316 last_pos = 0;
317 LIST_FOR_EACH_ENTRY(item, pool->item_list, link) {
318 if (src != dst || item->start_in_dw != last_pos) {
319 assert(last_pos <= item->start_in_dw);
320
321 compute_memory_move_item(pool, src, dst,
322 item, last_pos, pipe);
323 }
324
325 last_pos += align(item->size_in_dw, ITEM_ALIGNMENT);
326 }
327
328 pool->status &= ~POOL_FRAGMENTED;
329 }
330
331 /**
332 * Moves an item from the \a unallocated_list to the \a item_list.
333 * \param item The item that will be promoted.
334 * \return -1 if it fails, 0 otherwise
335 * \see compute_memory_finalize_pending
336 */
compute_memory_promote_item(struct compute_memory_pool * pool,struct compute_memory_item * item,struct pipe_context * pipe,int64_t start_in_dw)337 static int compute_memory_promote_item(struct compute_memory_pool *pool,
338 struct compute_memory_item *item, struct pipe_context *pipe,
339 int64_t start_in_dw)
340 {
341 struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
342 struct r600_context *rctx = (struct r600_context *)pipe;
343 struct pipe_resource *src = (struct pipe_resource *)item->real_buffer;
344 struct pipe_resource *dst = (struct pipe_resource *)pool->bo;
345 struct pipe_box box;
346
347 COMPUTE_DBG(pool->screen, "* compute_memory_promote_item()\n"
348 " + Promoting Item: %"PRIi64" , starting at: %"PRIi64" (%"PRIi64" bytes) "
349 "size: %"PRIi64" (%"PRIi64" bytes)\n\t\t\tnew start: %"PRIi64" (%"PRIi64" bytes)\n",
350 item->id, item->start_in_dw, item->start_in_dw * 4,
351 item->size_in_dw, item->size_in_dw * 4,
352 start_in_dw, start_in_dw * 4);
353
354 /* Remove the item from the unallocated list */
355 list_del(&item->link);
356
357 /* Add it back to the item_list */
358 list_addtail(&item->link, pool->item_list);
359 item->start_in_dw = start_in_dw;
360
361 if (src) {
362 u_box_1d(0, item->size_in_dw * 4, &box);
363
364 rctx->b.b.resource_copy_region(pipe,
365 dst, 0, item->start_in_dw * 4, 0 ,0,
366 src, 0, &box);
367
368 /* We check if the item is mapped for reading.
369 * In this case, we need to keep the temporary buffer 'alive'
370 * because it is possible to keep a map active for reading
371 * while a kernel (that reads from it) executes */
372 if (!(item->status & ITEM_MAPPED_FOR_READING) && !is_item_user_ptr(item)) {
373 pool->screen->b.b.resource_destroy(screen, src);
374 item->real_buffer = NULL;
375 }
376 }
377
378 return 0;
379 }
380
381 /**
382 * Moves an item from the \a item_list to the \a unallocated_list.
383 * \param item The item that will be demoted
384 * \see r600_compute_global_transfer_map
385 */
compute_memory_demote_item(struct compute_memory_pool * pool,struct compute_memory_item * item,struct pipe_context * pipe)386 void compute_memory_demote_item(struct compute_memory_pool *pool,
387 struct compute_memory_item *item, struct pipe_context *pipe)
388 {
389 struct r600_context *rctx = (struct r600_context *)pipe;
390 struct pipe_resource *src = (struct pipe_resource *)pool->bo;
391 struct pipe_resource *dst;
392 struct pipe_box box;
393
394 COMPUTE_DBG(pool->screen, "* compute_memory_demote_item()\n"
395 " + Demoting Item: %"PRIi64", starting at: %"PRIi64" (%"PRIi64" bytes) "
396 "size: %"PRIi64" (%"PRIi64" bytes)\n", item->id, item->start_in_dw,
397 item->start_in_dw * 4, item->size_in_dw, item->size_in_dw * 4);
398
399 /* First, we remove the item from the item_list */
400 list_del(&item->link);
401
402 /* Now we add it to the unallocated list */
403 list_addtail(&item->link, pool->unallocated_list);
404
405 /* We check if the intermediate buffer exists, and if it
406 * doesn't, we create it again */
407 if (item->real_buffer == NULL) {
408 item->real_buffer = r600_compute_buffer_alloc_vram(
409 pool->screen, item->size_in_dw * 4);
410 }
411
412 dst = (struct pipe_resource *)item->real_buffer;
413
414 /* We transfer the memory from the item in the pool to the
415 * temporary buffer. Download is skipped for items:
416 * - Not mapped for reading or writing (PIPE_MAP_DISCARD_RANGE).
417 * - Not writable by the device. */
418 if ((item->status & (ITEM_MAPPED_FOR_READING|ITEM_MAPPED_FOR_WRITING))) {
419
420 u_box_1d(item->start_in_dw * 4, item->size_in_dw * 4, &box);
421
422 rctx->b.b.resource_copy_region(pipe,
423 dst, 0, 0, 0, 0,
424 src, 0, &box);
425 }
426
427 /* Remember to mark the buffer as 'pending' by setting start_in_dw to -1 */
428 item->start_in_dw = -1;
429
430 if (item->link.next != pool->item_list) {
431 pool->status |= POOL_FRAGMENTED;
432 }
433 }
434
435 /**
436 * Moves the item \a item forward from the resource \a src to the
437 * resource \a dst at \a new_start_in_dw
438 *
439 * This function assumes two things:
440 * 1) The item is \b only moved forward, unless src is different from dst
441 * 2) The item \b won't change it's position inside the \a item_list
442 *
443 * \param item The item that will be moved
444 * \param new_start_in_dw The new position of the item in \a item_list
445 * \see compute_memory_defrag
446 */
compute_memory_move_item(struct compute_memory_pool * pool,struct pipe_resource * src,struct pipe_resource * dst,struct compute_memory_item * item,uint64_t new_start_in_dw,struct pipe_context * pipe)447 static void compute_memory_move_item(struct compute_memory_pool *pool,
448 struct pipe_resource *src, struct pipe_resource *dst,
449 struct compute_memory_item *item, uint64_t new_start_in_dw,
450 struct pipe_context *pipe)
451 {
452 struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
453 struct r600_context *rctx = (struct r600_context *)pipe;
454 struct pipe_box box;
455
456 COMPUTE_DBG(pool->screen, "* compute_memory_move_item()\n"
457 " + Moving item %"PRIi64" from %"PRIi64" (%"PRIi64" bytes) to %"PRIu64" (%"PRIu64" bytes)\n",
458 item->id, item->start_in_dw, item->start_in_dw * 4,
459 new_start_in_dw, new_start_in_dw * 4);
460
461 if (pool->item_list != item->link.prev) {
462 ASSERTED struct compute_memory_item *prev;
463 prev = container_of(item->link.prev, struct compute_memory_item, link);
464 assert(prev->start_in_dw + prev->size_in_dw <= new_start_in_dw);
465 }
466
467 u_box_1d(item->start_in_dw * 4, item->size_in_dw * 4, &box);
468
469 /* If the ranges don't overlap, or we are copying from one resource
470 * to another, we can just copy the item directly */
471 if (src != dst || new_start_in_dw + item->size_in_dw <= item->start_in_dw) {
472
473 rctx->b.b.resource_copy_region(pipe,
474 dst, 0, new_start_in_dw * 4, 0, 0,
475 src, 0, &box);
476 } else {
477 /* The ranges overlap, we will try first to use an intermediate
478 * resource to move the item */
479 struct pipe_resource *tmp = (struct pipe_resource *)
480 r600_compute_buffer_alloc_vram(pool->screen, item->size_in_dw * 4);
481
482 if (tmp != NULL) {
483 rctx->b.b.resource_copy_region(pipe,
484 tmp, 0, 0, 0, 0,
485 src, 0, &box);
486
487 box.x = 0;
488
489 rctx->b.b.resource_copy_region(pipe,
490 dst, 0, new_start_in_dw * 4, 0, 0,
491 tmp, 0, &box);
492
493 pool->screen->b.b.resource_destroy(screen, tmp);
494
495 } else {
496 /* The allocation of the temporary resource failed,
497 * falling back to use mappings */
498 uint32_t *map;
499 int64_t offset;
500 struct pipe_transfer *trans;
501
502 offset = item->start_in_dw - new_start_in_dw;
503
504 u_box_1d(new_start_in_dw * 4, (offset + item->size_in_dw) * 4, &box);
505
506 map = pipe->buffer_map(pipe, src, 0, PIPE_MAP_READ_WRITE,
507 &box, &trans);
508
509 assert(map);
510 assert(trans);
511
512 memmove(map, map + offset, item->size_in_dw * 4);
513
514 pipe->buffer_unmap(pipe, trans);
515 }
516 }
517
518 item->start_in_dw = new_start_in_dw;
519 }
520
521 /**
522 * Frees one item for compute_memory_free()
523 */
compute_memory_free_item(struct pipe_screen * screen,struct compute_memory_item * item)524 static void compute_memory_free_item(struct pipe_screen *screen,
525 struct compute_memory_item *item)
526 {
527 struct pipe_resource *res = (struct pipe_resource *)item->real_buffer;
528
529 list_del(&item->link);
530
531 if (res && !is_item_user_ptr(item))
532 screen->resource_destroy(screen, res);
533
534 free(item);
535 }
536
537 /**
538 * Frees the memory associated to the item with id \a id from the pool.
539 * \param id The id of the item to be freed.
540 */
compute_memory_free(struct compute_memory_pool * pool,int64_t id)541 void compute_memory_free(struct compute_memory_pool* pool, int64_t id)
542 {
543 struct compute_memory_item *item, *next;
544 struct pipe_screen *screen = (struct pipe_screen *)pool->screen;
545
546 COMPUTE_DBG(pool->screen, "* compute_memory_free() id + %"PRIi64" \n", id);
547
548 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->item_list, link) {
549 if (item->id == id) {
550 if (item->link.next != pool->item_list) {
551 pool->status |= POOL_FRAGMENTED;
552 }
553
554 compute_memory_free_item(screen, item);
555 return;
556 }
557 }
558
559 LIST_FOR_EACH_ENTRY_SAFE(item, next, pool->unallocated_list, link) {
560 if (item->id == id) {
561 compute_memory_free_item(screen, item);
562 return;
563 }
564 }
565
566 fprintf(stderr, "Internal error, invalid id %"PRIi64" "
567 "for compute_memory_free\n", id);
568
569 assert(0 && "error");
570 }
571
572 /**
573 * Creates pending allocations for new items, these items are
574 * placed in the unallocated_list.
575 * \param size_in_dw The size, in double words, of the new item.
576 * \return The new item
577 * \see r600_compute_global_buffer_create
578 */
compute_memory_alloc(struct compute_memory_pool * pool,int64_t size_in_dw)579 struct compute_memory_item* compute_memory_alloc(
580 struct compute_memory_pool* pool,
581 int64_t size_in_dw)
582 {
583 struct compute_memory_item *new_item = NULL;
584
585 COMPUTE_DBG(pool->screen, "* compute_memory_alloc() size_in_dw = %"PRIi64" (%"PRIi64" bytes)\n",
586 size_in_dw, 4 * size_in_dw);
587
588 new_item = (struct compute_memory_item *)
589 CALLOC(sizeof(struct compute_memory_item), 1);
590 if (!new_item)
591 return NULL;
592
593 new_item->size_in_dw = size_in_dw;
594 new_item->start_in_dw = -1; /* mark pending */
595 new_item->id = pool->next_id++;
596 new_item->pool = pool;
597 new_item->real_buffer = NULL;
598
599 list_addtail(&new_item->link, pool->unallocated_list);
600
601 COMPUTE_DBG(pool->screen, " + Adding item %p id = %"PRIi64" size = %"PRIi64" (%"PRIi64" bytes)\n",
602 new_item, new_item->id, new_item->size_in_dw,
603 new_item->size_in_dw * 4);
604 return new_item;
605 }
606
607 /**
608 * Transfer data host<->device, offset and size is in bytes.
609 * \param device_to_host 1 for device->host, 0 for host->device.
610 * \see compute_memory_shadow
611 */
compute_memory_transfer(struct compute_memory_pool * pool,struct pipe_context * pipe,int device_to_host,struct compute_memory_item * chunk,void * data,int offset_in_chunk,int size)612 static void compute_memory_transfer(
613 struct compute_memory_pool* pool,
614 struct pipe_context * pipe,
615 int device_to_host,
616 struct compute_memory_item* chunk,
617 void* data,
618 int offset_in_chunk,
619 int size)
620 {
621 int64_t aligned_size = pool->size_in_dw;
622 struct pipe_resource* gart = (struct pipe_resource*)pool->bo;
623 int64_t internal_offset = chunk->start_in_dw*4 + offset_in_chunk;
624
625 struct pipe_transfer *xfer;
626 uint32_t *map;
627
628 assert(gart);
629
630 COMPUTE_DBG(pool->screen, "* compute_memory_transfer() device_to_host = %d, "
631 "offset_in_chunk = %d, size = %d\n", device_to_host,
632 offset_in_chunk, size);
633
634 if (device_to_host) {
635 map = pipe->buffer_map(pipe, gart, 0, PIPE_MAP_READ,
636 &(struct pipe_box) { .width = aligned_size * 4,
637 .height = 1, .depth = 1 }, &xfer);
638 assert(xfer);
639 assert(map);
640 memcpy(data, map + internal_offset, size);
641 pipe->buffer_unmap(pipe, xfer);
642 } else {
643 map = pipe->buffer_map(pipe, gart, 0, PIPE_MAP_WRITE,
644 &(struct pipe_box) { .width = aligned_size * 4,
645 .height = 1, .depth = 1 }, &xfer);
646 assert(xfer);
647 assert(map);
648 memcpy(map + internal_offset, data, size);
649 pipe->buffer_unmap(pipe, xfer);
650 }
651 }
652