1 /*
2 * Copyright © 2011 Marek Olšák <[email protected]>
3 * Copyright © 2015 Advanced Micro Devices, Inc.
4 * Copyright © 2021 Valve Corporation
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining
8 * a copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
17 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
19 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
26 * of the Software.
27 *
28 * Authors:
29 * Mike Blumenkrantz <[email protected]>
30 */
31
32 #include "zink_context.h"
33 #include "zink_bo.h"
34 #include "zink_resource.h"
35 #include "zink_screen.h"
36 #include "util/u_hash_table.h"
37
38 #if !defined(__APPLE__) && !defined(_WIN32)
39 #define ZINK_USE_DMABUF
40 #include <xf86drm.h>
41 #endif
42
43 struct zink_bo;
44
45 struct zink_sparse_backing_chunk {
46 uint32_t begin, end;
47 };
48
49
50 /*
51 * Sub-allocation information for a real buffer used as backing memory of a
52 * sparse buffer.
53 */
54 struct zink_sparse_backing {
55 struct list_head list;
56
57 struct zink_bo *bo;
58
59 /* Sorted list of free chunks. */
60 struct zink_sparse_backing_chunk *chunks;
61 uint32_t max_chunks;
62 uint32_t num_chunks;
63 };
64
65 struct zink_sparse_commitment {
66 struct zink_sparse_backing *backing;
67 uint32_t page;
68 };
69
70 struct zink_slab {
71 struct pb_slab base;
72 struct zink_bo *buffer;
73 struct zink_bo *entries;
74 };
75
76
77 ALWAYS_INLINE static struct zink_slab *
zink_slab(struct pb_slab * pslab)78 zink_slab(struct pb_slab *pslab)
79 {
80 return (struct zink_slab*)pslab;
81 }
82
83 static struct pb_slabs *
get_slabs(struct zink_screen * screen,uint64_t size,enum zink_alloc_flag flags)84 get_slabs(struct zink_screen *screen, uint64_t size, enum zink_alloc_flag flags)
85 {
86 //struct pb_slabs *bo_slabs = ((flags & RADEON_FLAG_ENCRYPTED) && screen->info.has_tmz_support) ?
87 //screen->bo_slabs_encrypted : screen->bo_slabs;
88
89 struct pb_slabs *bo_slabs = screen->pb.bo_slabs;
90 /* Find the correct slab allocator for the given size. */
91 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
92 struct pb_slabs *slabs = &bo_slabs[i];
93
94 if (size <= 1ULL << (slabs->min_order + slabs->num_orders - 1))
95 return slabs;
96 }
97
98 assert(0);
99 return NULL;
100 }
101
102 /* Return the power of two size of a slab entry matching the input size. */
103 static unsigned
get_slab_pot_entry_size(struct zink_screen * screen,unsigned size)104 get_slab_pot_entry_size(struct zink_screen *screen, unsigned size)
105 {
106 unsigned entry_size = util_next_power_of_two(size);
107 unsigned min_entry_size = 1 << screen->pb.bo_slabs[0].min_order;
108
109 return MAX2(entry_size, min_entry_size);
110 }
111
112 /* Return the slab entry alignment. */
get_slab_entry_alignment(struct zink_screen * screen,unsigned size)113 static unsigned get_slab_entry_alignment(struct zink_screen *screen, unsigned size)
114 {
115 unsigned entry_size = get_slab_pot_entry_size(screen, size);
116
117 if (size <= entry_size * 3 / 4)
118 return entry_size / 4;
119
120 return entry_size;
121 }
122
123 static void
bo_destroy(struct zink_screen * screen,struct pb_buffer * pbuf)124 bo_destroy(struct zink_screen *screen, struct pb_buffer *pbuf)
125 {
126 struct zink_bo *bo = zink_bo(pbuf);
127
128 #ifdef ZINK_USE_DMABUF
129 if (bo->mem && !bo->u.real.use_reusable_pool) {
130 simple_mtx_lock(&bo->u.real.export_lock);
131 list_for_each_entry_safe(struct bo_export, export, &bo->u.real.exports, link) {
132 struct drm_gem_close args = { .handle = export->gem_handle };
133 drmIoctl(export->drm_fd, DRM_IOCTL_GEM_CLOSE, &args);
134 list_del(&export->link);
135 free(export);
136 }
137 simple_mtx_unlock(&bo->u.real.export_lock);
138 simple_mtx_destroy(&bo->u.real.export_lock);
139 }
140 #endif
141
142 if (!bo->u.real.is_user_ptr && bo->u.real.cpu_ptr) {
143 bo->u.real.map_count = 1;
144 bo->u.real.cpu_ptr = NULL;
145 zink_bo_unmap(screen, bo);
146 }
147
148 VKSCR(FreeMemory)(screen->dev, bo->mem, NULL);
149
150 simple_mtx_destroy(&bo->lock);
151 FREE(bo);
152 }
153
154 static bool
bo_can_reclaim(struct zink_screen * screen,struct pb_buffer * pbuf)155 bo_can_reclaim(struct zink_screen *screen, struct pb_buffer *pbuf)
156 {
157 struct zink_bo *bo = zink_bo(pbuf);
158
159 return zink_screen_usage_check_completion(screen, bo->reads.u) && zink_screen_usage_check_completion(screen, bo->writes.u);
160 }
161
162 static bool
bo_can_reclaim_slab(void * priv,struct pb_slab_entry * entry)163 bo_can_reclaim_slab(void *priv, struct pb_slab_entry *entry)
164 {
165 struct zink_bo *bo = container_of(entry, struct zink_bo, u.slab.entry);
166
167 return bo_can_reclaim(priv, &bo->base);
168 }
169
170 static void
bo_slab_free(struct zink_screen * screen,struct pb_slab * pslab)171 bo_slab_free(struct zink_screen *screen, struct pb_slab *pslab)
172 {
173 struct zink_slab *slab = zink_slab(pslab);
174 ASSERTED unsigned slab_size = slab->buffer->base.base.size;
175
176 assert(slab->base.num_entries * slab->base.entry_size <= slab_size);
177 FREE(slab->entries);
178 zink_bo_unref(screen, slab->buffer);
179 FREE(slab);
180 }
181
182 static void
bo_slab_destroy(struct zink_screen * screen,struct pb_buffer * pbuf)183 bo_slab_destroy(struct zink_screen *screen, struct pb_buffer *pbuf)
184 {
185 struct zink_bo *bo = zink_bo(pbuf);
186
187 assert(!bo->mem);
188
189 //if (bo->base.usage & RADEON_FLAG_ENCRYPTED)
190 //pb_slab_free(get_slabs(screen, bo->base.size, RADEON_FLAG_ENCRYPTED), &bo->u.slab.entry);
191 //else
192 pb_slab_free(get_slabs(screen, bo->base.base.size, 0), &bo->u.slab.entry);
193 }
194
195 static bool
clean_up_buffer_managers(struct zink_screen * screen)196 clean_up_buffer_managers(struct zink_screen *screen)
197 {
198 unsigned num_reclaims = 0;
199 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
200 num_reclaims += pb_slabs_reclaim(&screen->pb.bo_slabs[i]);
201 //if (screen->info.has_tmz_support)
202 //pb_slabs_reclaim(&screen->bo_slabs_encrypted[i]);
203 }
204
205 num_reclaims += pb_cache_release_all_buffers(&screen->pb.bo_cache);
206 return !!num_reclaims;
207 }
208
209 static unsigned
get_optimal_alignment(struct zink_screen * screen,uint64_t size,unsigned alignment)210 get_optimal_alignment(struct zink_screen *screen, uint64_t size, unsigned alignment)
211 {
212 /* Increase the alignment for faster address translation and better memory
213 * access pattern.
214 */
215 if (size >= 4096) {
216 alignment = MAX2(alignment, 4096);
217 } else if (size) {
218 unsigned msb = util_last_bit(size);
219
220 alignment = MAX2(alignment, 1u << (msb - 1));
221 }
222 return alignment;
223 }
224
225 static void
bo_destroy_or_cache(struct zink_screen * screen,struct pb_buffer * pbuf)226 bo_destroy_or_cache(struct zink_screen *screen, struct pb_buffer *pbuf)
227 {
228 struct zink_bo *bo = zink_bo(pbuf);
229
230 assert(bo->mem); /* slab buffers have a separate vtbl */
231 bo->reads.u = NULL;
232 bo->writes.u = NULL;
233
234 if (bo->u.real.use_reusable_pool)
235 pb_cache_add_buffer(&screen->pb.bo_cache, bo->cache_entry);
236 else
237 bo_destroy(screen, pbuf);
238 }
239
240 static const struct pb_vtbl bo_vtbl = {
241 /* Cast to void* because one of the function parameters is a struct pointer instead of void*. */
242 (void*)bo_destroy_or_cache
243 /* other functions are never called */
244 };
245
246 static struct zink_bo *
bo_create_internal(struct zink_screen * screen,uint64_t size,unsigned alignment,enum zink_heap heap,unsigned mem_type_idx,unsigned flags,const void * pNext)247 bo_create_internal(struct zink_screen *screen,
248 uint64_t size,
249 unsigned alignment,
250 enum zink_heap heap,
251 unsigned mem_type_idx,
252 unsigned flags,
253 const void *pNext)
254 {
255 struct zink_bo *bo = NULL;
256 bool init_pb_cache;
257
258 alignment = get_optimal_alignment(screen, size, alignment);
259
260 VkMemoryAllocateFlagsInfo ai;
261 ai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO;
262 ai.pNext = pNext;
263 ai.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
264 ai.deviceMask = 0;
265 if (screen->info.have_KHR_buffer_device_address)
266 pNext = &ai;
267
268 VkMemoryPriorityAllocateInfoEXT prio = {
269 VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT,
270 pNext,
271 (flags & ZINK_ALLOC_NO_SUBALLOC) ? 1.0 : 0.5,
272 };
273 if (screen->info.have_EXT_memory_priority)
274 pNext = &prio;
275
276 VkMemoryAllocateInfo mai;
277 mai.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
278 mai.pNext = pNext;
279 mai.allocationSize = size;
280 mai.memoryTypeIndex = mem_type_idx;
281 if (screen->info.mem_props.memoryTypes[mai.memoryTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) {
282 alignment = MAX2(alignment, screen->info.props.limits.minMemoryMapAlignment);
283 mai.allocationSize = align64(mai.allocationSize, screen->info.props.limits.minMemoryMapAlignment);
284 }
285 unsigned vk_heap_idx = screen->info.mem_props.memoryTypes[mem_type_idx].heapIndex;
286 if (mai.allocationSize > screen->info.mem_props.memoryHeaps[vk_heap_idx].size) {
287 mesa_loge("zink: can't allocate %"PRIu64" bytes from heap that's only %"PRIu64" bytes!\n", mai.allocationSize, screen->info.mem_props.memoryHeaps[vk_heap_idx].size);
288 return NULL;
289 }
290
291 /* all non-suballocated bo can cache */
292 init_pb_cache = !pNext;
293
294 if (!bo)
295 bo = CALLOC(1, sizeof(struct zink_bo) + init_pb_cache * sizeof(struct pb_cache_entry));
296 if (!bo) {
297 return NULL;
298 }
299
300 VkResult ret = VKSCR(AllocateMemory)(screen->dev, &mai, NULL, &bo->mem);
301 if (!zink_screen_handle_vkresult(screen, ret)) {
302 mesa_loge("zink: couldn't allocate memory: heap=%u size=%" PRIu64, heap, size);
303 if (zink_debug & ZINK_DEBUG_MEM) {
304 zink_debug_mem_print_stats(screen);
305 /* abort with mem debug to allow debugging */
306 abort();
307 }
308 goto fail;
309 }
310
311 if (init_pb_cache) {
312 bo->u.real.use_reusable_pool = true;
313 pb_cache_init_entry(&screen->pb.bo_cache, bo->cache_entry, &bo->base.base, mem_type_idx);
314 } else {
315 #ifdef ZINK_USE_DMABUF
316 list_inithead(&bo->u.real.exports);
317 simple_mtx_init(&bo->u.real.export_lock, mtx_plain);
318 #endif
319 }
320
321
322 simple_mtx_init(&bo->lock, mtx_plain);
323 pipe_reference_init(&bo->base.base.reference, 1);
324 bo->base.base.alignment_log2 = util_logbase2(alignment);
325 bo->base.base.size = mai.allocationSize;
326 bo->base.vtbl = &bo_vtbl;
327 bo->base.base.placement = mem_type_idx;
328 bo->base.base.usage = flags;
329
330 return bo;
331
332 fail:
333 bo_destroy(screen, (void*)bo);
334 return NULL;
335 }
336
337 /*
338 * Attempt to allocate the given number of backing pages. Fewer pages may be
339 * allocated (depending on the fragmentation of existing backing buffers),
340 * which will be reflected by a change to *pnum_pages.
341 */
342 static struct zink_sparse_backing *
sparse_backing_alloc(struct zink_screen * screen,struct zink_bo * bo,uint32_t * pstart_page,uint32_t * pnum_pages)343 sparse_backing_alloc(struct zink_screen *screen, struct zink_bo *bo,
344 uint32_t *pstart_page, uint32_t *pnum_pages)
345 {
346 struct zink_sparse_backing *best_backing;
347 unsigned best_idx;
348 uint32_t best_num_pages;
349
350 best_backing = NULL;
351 best_idx = 0;
352 best_num_pages = 0;
353
354 /* This is a very simple and inefficient best-fit algorithm. */
355 list_for_each_entry(struct zink_sparse_backing, backing, &bo->u.sparse.backing, list) {
356 for (unsigned idx = 0; idx < backing->num_chunks; ++idx) {
357 uint32_t cur_num_pages = backing->chunks[idx].end - backing->chunks[idx].begin;
358 if ((best_num_pages < *pnum_pages && cur_num_pages > best_num_pages) ||
359 (best_num_pages > *pnum_pages && cur_num_pages < best_num_pages)) {
360 best_backing = backing;
361 best_idx = idx;
362 best_num_pages = cur_num_pages;
363 }
364 }
365 }
366
367 /* Allocate a new backing buffer if necessary. */
368 if (!best_backing) {
369 struct pb_buffer *buf;
370 uint64_t size;
371 uint32_t pages;
372
373 best_backing = CALLOC_STRUCT(zink_sparse_backing);
374 if (!best_backing)
375 return NULL;
376
377 best_backing->max_chunks = 4;
378 best_backing->chunks = CALLOC(best_backing->max_chunks,
379 sizeof(*best_backing->chunks));
380 if (!best_backing->chunks) {
381 FREE(best_backing);
382 return NULL;
383 }
384
385 assert(bo->u.sparse.num_backing_pages < DIV_ROUND_UP(bo->base.base.size, ZINK_SPARSE_BUFFER_PAGE_SIZE));
386
387 size = MIN3(bo->base.base.size / 16,
388 8 * 1024 * 1024,
389 bo->base.base.size - (uint64_t)bo->u.sparse.num_backing_pages * ZINK_SPARSE_BUFFER_PAGE_SIZE);
390 size = MAX2(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
391
392 buf = zink_bo_create(screen, size, ZINK_SPARSE_BUFFER_PAGE_SIZE,
393 ZINK_HEAP_DEVICE_LOCAL, 0, screen->heap_map[ZINK_HEAP_DEVICE_LOCAL][0], NULL);
394 if (!buf) {
395 FREE(best_backing->chunks);
396 FREE(best_backing);
397 return NULL;
398 }
399
400 /* We might have gotten a bigger buffer than requested via caching. */
401 pages = buf->base.size / ZINK_SPARSE_BUFFER_PAGE_SIZE;
402
403 best_backing->bo = zink_bo(buf);
404 best_backing->num_chunks = 1;
405 best_backing->chunks[0].begin = 0;
406 best_backing->chunks[0].end = pages;
407
408 list_add(&best_backing->list, &bo->u.sparse.backing);
409 bo->u.sparse.num_backing_pages += pages;
410
411 best_idx = 0;
412 best_num_pages = pages;
413 }
414
415 *pnum_pages = MIN2(*pnum_pages, best_num_pages);
416 *pstart_page = best_backing->chunks[best_idx].begin;
417 best_backing->chunks[best_idx].begin += *pnum_pages;
418
419 if (best_backing->chunks[best_idx].begin >= best_backing->chunks[best_idx].end) {
420 memmove(&best_backing->chunks[best_idx], &best_backing->chunks[best_idx + 1],
421 sizeof(*best_backing->chunks) * (best_backing->num_chunks - best_idx - 1));
422 best_backing->num_chunks--;
423 }
424
425 return best_backing;
426 }
427
428 static void
sparse_free_backing_buffer(struct zink_screen * screen,struct zink_bo * bo,struct zink_sparse_backing * backing)429 sparse_free_backing_buffer(struct zink_screen *screen, struct zink_bo *bo,
430 struct zink_sparse_backing *backing)
431 {
432 bo->u.sparse.num_backing_pages -= backing->bo->base.base.size / ZINK_SPARSE_BUFFER_PAGE_SIZE;
433
434 list_del(&backing->list);
435 zink_bo_unref(screen, backing->bo);
436 FREE(backing->chunks);
437 FREE(backing);
438 }
439
440 /*
441 * Return a range of pages from the given backing buffer back into the
442 * free structure.
443 */
444 static bool
sparse_backing_free(struct zink_screen * screen,struct zink_bo * bo,struct zink_sparse_backing * backing,uint32_t start_page,uint32_t num_pages)445 sparse_backing_free(struct zink_screen *screen, struct zink_bo *bo,
446 struct zink_sparse_backing *backing,
447 uint32_t start_page, uint32_t num_pages)
448 {
449 uint32_t end_page = start_page + num_pages;
450 unsigned low = 0;
451 unsigned high = backing->num_chunks;
452
453 /* Find the first chunk with begin >= start_page. */
454 while (low < high) {
455 unsigned mid = low + (high - low) / 2;
456
457 if (backing->chunks[mid].begin >= start_page)
458 high = mid;
459 else
460 low = mid + 1;
461 }
462
463 assert(low >= backing->num_chunks || end_page <= backing->chunks[low].begin);
464 assert(low == 0 || backing->chunks[low - 1].end <= start_page);
465
466 if (low > 0 && backing->chunks[low - 1].end == start_page) {
467 backing->chunks[low - 1].end = end_page;
468
469 if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
470 backing->chunks[low - 1].end = backing->chunks[low].end;
471 memmove(&backing->chunks[low], &backing->chunks[low + 1],
472 sizeof(*backing->chunks) * (backing->num_chunks - low - 1));
473 backing->num_chunks--;
474 }
475 } else if (low < backing->num_chunks && end_page == backing->chunks[low].begin) {
476 backing->chunks[low].begin = start_page;
477 } else {
478 if (backing->num_chunks >= backing->max_chunks) {
479 unsigned new_max_chunks = 2 * backing->max_chunks;
480 struct zink_sparse_backing_chunk *new_chunks =
481 REALLOC(backing->chunks,
482 sizeof(*backing->chunks) * backing->max_chunks,
483 sizeof(*backing->chunks) * new_max_chunks);
484 if (!new_chunks)
485 return false;
486
487 backing->max_chunks = new_max_chunks;
488 backing->chunks = new_chunks;
489 }
490
491 memmove(&backing->chunks[low + 1], &backing->chunks[low],
492 sizeof(*backing->chunks) * (backing->num_chunks - low));
493 backing->chunks[low].begin = start_page;
494 backing->chunks[low].end = end_page;
495 backing->num_chunks++;
496 }
497
498 if (backing->num_chunks == 1 && backing->chunks[0].begin == 0 &&
499 backing->chunks[0].end == backing->bo->base.base.size / ZINK_SPARSE_BUFFER_PAGE_SIZE)
500 sparse_free_backing_buffer(screen, bo, backing);
501
502 return true;
503 }
504
505 static void
bo_sparse_destroy(struct zink_screen * screen,struct pb_buffer * pbuf)506 bo_sparse_destroy(struct zink_screen *screen, struct pb_buffer *pbuf)
507 {
508 struct zink_bo *bo = zink_bo(pbuf);
509
510 assert(!bo->mem && bo->base.base.usage & ZINK_ALLOC_SPARSE);
511
512 while (!list_is_empty(&bo->u.sparse.backing)) {
513 sparse_free_backing_buffer(screen, bo,
514 container_of(bo->u.sparse.backing.next,
515 struct zink_sparse_backing, list));
516 }
517
518 FREE(bo->u.sparse.commitments);
519 simple_mtx_destroy(&bo->lock);
520 FREE(bo);
521 }
522
523 static const struct pb_vtbl bo_sparse_vtbl = {
524 /* Cast to void* because one of the function parameters is a struct pointer instead of void*. */
525 (void*)bo_sparse_destroy
526 /* other functions are never called */
527 };
528
529 static struct pb_buffer *
bo_sparse_create(struct zink_screen * screen,uint64_t size)530 bo_sparse_create(struct zink_screen *screen, uint64_t size)
531 {
532 struct zink_bo *bo;
533
534 /* We use 32-bit page numbers; refuse to attempt allocating sparse buffers
535 * that exceed this limit. This is not really a restriction: we don't have
536 * that much virtual address space anyway.
537 */
538 if (size > (uint64_t)INT32_MAX * ZINK_SPARSE_BUFFER_PAGE_SIZE)
539 return NULL;
540
541 bo = CALLOC_STRUCT(zink_bo);
542 if (!bo)
543 return NULL;
544
545 simple_mtx_init(&bo->lock, mtx_plain);
546 pipe_reference_init(&bo->base.base.reference, 1);
547 bo->base.base.alignment_log2 = util_logbase2(ZINK_SPARSE_BUFFER_PAGE_SIZE);
548 bo->base.base.size = size;
549 bo->base.vtbl = &bo_sparse_vtbl;
550 unsigned placement = zink_mem_type_idx_from_types(screen, ZINK_HEAP_DEVICE_LOCAL_SPARSE, UINT32_MAX);
551 assert(placement != UINT32_MAX);
552 bo->base.base.placement = placement;
553 bo->unique_id = p_atomic_inc_return(&screen->pb.next_bo_unique_id);
554 bo->base.base.usage = ZINK_ALLOC_SPARSE;
555
556 bo->u.sparse.num_va_pages = DIV_ROUND_UP(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
557 bo->u.sparse.commitments = CALLOC(bo->u.sparse.num_va_pages,
558 sizeof(*bo->u.sparse.commitments));
559 if (!bo->u.sparse.commitments)
560 goto error_alloc_commitments;
561
562 list_inithead(&bo->u.sparse.backing);
563
564 return &bo->base;
565
566 error_alloc_commitments:
567 simple_mtx_destroy(&bo->lock);
568 FREE(bo);
569 return NULL;
570 }
571
572 struct pb_buffer *
zink_bo_create(struct zink_screen * screen,uint64_t size,unsigned alignment,enum zink_heap heap,enum zink_alloc_flag flags,unsigned mem_type_idx,const void * pNext)573 zink_bo_create(struct zink_screen *screen, uint64_t size, unsigned alignment, enum zink_heap heap, enum zink_alloc_flag flags, unsigned mem_type_idx, const void *pNext)
574 {
575 struct zink_bo *bo;
576 /* pull in sparse flag */
577 flags |= zink_alloc_flags_from_heap(heap);
578
579 //struct pb_slabs *slabs = ((flags & RADEON_FLAG_ENCRYPTED) && screen->info.has_tmz_support) ?
580 //screen->bo_slabs_encrypted : screen->bo_slabs;
581 struct pb_slabs *bo_slabs = screen->pb.bo_slabs;
582
583 struct pb_slabs *last_slab = &bo_slabs[NUM_SLAB_ALLOCATORS - 1];
584 unsigned max_slab_entry_size = 1 << (last_slab->min_order + last_slab->num_orders - 1);
585
586 /* Sub-allocate small buffers from slabs. */
587 if (!(flags & (ZINK_ALLOC_NO_SUBALLOC | ZINK_ALLOC_SPARSE)) &&
588 size <= max_slab_entry_size) {
589 struct pb_slab_entry *entry;
590
591 if (heap < 0 || heap >= ZINK_HEAP_MAX)
592 goto no_slab;
593
594 unsigned alloc_size = size;
595
596 /* Always use slabs for sizes less than 4 KB because the kernel aligns
597 * everything to 4 KB.
598 */
599 if (size < alignment && alignment <= 4 * 1024)
600 alloc_size = alignment;
601
602 if (alignment > get_slab_entry_alignment(screen, alloc_size)) {
603 /* 3/4 allocations can return too small alignment. Try again with a power of two
604 * allocation size.
605 */
606 unsigned pot_size = get_slab_pot_entry_size(screen, alloc_size);
607
608 if (alignment <= pot_size) {
609 /* This size works but wastes some memory to fulfil the alignment. */
610 alloc_size = pot_size;
611 } else {
612 goto no_slab; /* can't fulfil alignment requirements */
613 }
614 }
615
616 struct pb_slabs *slabs = get_slabs(screen, alloc_size, flags);
617 bool reclaim_all = false;
618 if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE && !screen->resizable_bar) {
619 unsigned low_bound = 128 * 1024 * 1024; //128MB is a very small BAR
620 if (zink_driverid(screen) == VK_DRIVER_ID_NVIDIA_PROPRIETARY)
621 low_bound *= 2; //nvidia has fat textures or something
622 unsigned vk_heap_idx = screen->info.mem_props.memoryTypes[mem_type_idx].heapIndex;
623 reclaim_all = screen->info.mem_props.memoryHeaps[vk_heap_idx].size <= low_bound;
624 if (reclaim_all)
625 reclaim_all = clean_up_buffer_managers(screen);
626 }
627 entry = pb_slab_alloc_reclaimed(slabs, alloc_size, mem_type_idx, reclaim_all);
628 if (!entry) {
629 /* Clean up buffer managers and try again. */
630 if (clean_up_buffer_managers(screen))
631 entry = pb_slab_alloc_reclaimed(slabs, alloc_size, mem_type_idx, true);
632 }
633 if (!entry)
634 return NULL;
635
636 bo = container_of(entry, struct zink_bo, u.slab.entry);
637 assert(bo->base.base.placement == mem_type_idx);
638 pipe_reference_init(&bo->base.base.reference, 1);
639 bo->base.base.size = size;
640 memset(&bo->reads, 0, sizeof(bo->reads));
641 memset(&bo->writes, 0, sizeof(bo->writes));
642 bo->unique_id = p_atomic_inc_return(&screen->pb.next_bo_unique_id);
643 assert(alignment <= 1 << bo->base.base.alignment_log2);
644
645 return &bo->base;
646 }
647 no_slab:
648
649 if (flags & ZINK_ALLOC_SPARSE) {
650 assert(ZINK_SPARSE_BUFFER_PAGE_SIZE % alignment == 0);
651
652 return bo_sparse_create(screen, size);
653 }
654
655 /* Align size to page size. This is the minimum alignment for normal
656 * BOs. Aligning this here helps the cached bufmgr. Especially small BOs,
657 * like constant/uniform buffers, can benefit from better and more reuse.
658 */
659 if (heap == ZINK_HEAP_DEVICE_LOCAL_VISIBLE) {
660 size = align64(size, screen->info.props.limits.minMemoryMapAlignment);
661 alignment = align(alignment, screen->info.props.limits.minMemoryMapAlignment);
662 }
663
664 bool use_reusable_pool = !(flags & ZINK_ALLOC_NO_SUBALLOC);
665
666 if (use_reusable_pool) {
667 /* Get a buffer from the cache. */
668 bo = (struct zink_bo*)
669 pb_cache_reclaim_buffer(&screen->pb.bo_cache, size, alignment, 0, mem_type_idx);
670 assert(!bo || bo->base.base.placement == mem_type_idx);
671 if (bo) {
672 memset(&bo->reads, 0, sizeof(bo->reads));
673 memset(&bo->writes, 0, sizeof(bo->writes));
674 return &bo->base;
675 }
676 }
677
678 /* Create a new one. */
679 bo = bo_create_internal(screen, size, alignment, heap, mem_type_idx, flags, pNext);
680 if (!bo) {
681 /* Clean up buffer managers and try again. */
682 if (clean_up_buffer_managers(screen))
683 bo = bo_create_internal(screen, size, alignment, heap, mem_type_idx, flags, pNext);
684 if (!bo)
685 return NULL;
686 }
687 assert(bo->base.base.placement == mem_type_idx);
688
689 return &bo->base;
690 }
691
692 void *
zink_bo_map(struct zink_screen * screen,struct zink_bo * bo)693 zink_bo_map(struct zink_screen *screen, struct zink_bo *bo)
694 {
695 void *cpu = NULL;
696 uint64_t offset = 0;
697 struct zink_bo *real;
698
699 if (bo->mem) {
700 real = bo;
701 } else {
702 real = bo->u.slab.real;
703 offset = bo->offset - real->offset;
704 }
705
706 cpu = p_atomic_read(&real->u.real.cpu_ptr);
707 if (!cpu) {
708 simple_mtx_lock(&real->lock);
709 /* Must re-check due to the possibility of a race. Re-check need not
710 * be atomic thanks to the lock. */
711 cpu = real->u.real.cpu_ptr;
712 if (!cpu) {
713 VkResult result = VKSCR(MapMemory)(screen->dev, real->mem, 0, real->base.base.size, 0, &cpu);
714 if (result != VK_SUCCESS) {
715 mesa_loge("ZINK: vkMapMemory failed (%s)", vk_Result_to_str(result));
716 simple_mtx_unlock(&real->lock);
717 return NULL;
718 }
719 if (unlikely(zink_debug & ZINK_DEBUG_MAP)) {
720 p_atomic_add(&screen->mapped_vram, real->base.base.size);
721 mesa_loge("NEW MAP(%"PRIu64") TOTAL(%"PRIu64")", real->base.base.size, screen->mapped_vram);
722 }
723 p_atomic_set(&real->u.real.cpu_ptr, cpu);
724 }
725 simple_mtx_unlock(&real->lock);
726 }
727 p_atomic_inc(&real->u.real.map_count);
728
729 return (uint8_t*)cpu + offset;
730 }
731
732 void
zink_bo_unmap(struct zink_screen * screen,struct zink_bo * bo)733 zink_bo_unmap(struct zink_screen *screen, struct zink_bo *bo)
734 {
735 struct zink_bo *real = bo->mem ? bo : bo->u.slab.real;
736
737 assert(real->u.real.map_count != 0 && "too many unmaps");
738
739 if (p_atomic_dec_zero(&real->u.real.map_count)) {
740 p_atomic_set(&real->u.real.cpu_ptr, NULL);
741 if (unlikely(zink_debug & ZINK_DEBUG_MAP)) {
742 p_atomic_add(&screen->mapped_vram, -real->base.base.size);
743 mesa_loge("UNMAP(%"PRIu64") TOTAL(%"PRIu64")", real->base.base.size, screen->mapped_vram);
744 }
745 VKSCR(UnmapMemory)(screen->dev, real->mem);
746 }
747 }
748
749 /* see comment in zink_batch_reference_resource_move for how references on sparse backing buffers are organized */
750 static void
track_freed_sparse_bo(struct zink_context * ctx,struct zink_sparse_backing * backing)751 track_freed_sparse_bo(struct zink_context *ctx, struct zink_sparse_backing *backing)
752 {
753 pipe_reference(NULL, &backing->bo->base.base.reference);
754 util_dynarray_append(&ctx->bs->freed_sparse_backing_bos, struct zink_bo*, backing->bo);
755 }
756
757 static VkSemaphore
buffer_commit_single(struct zink_screen * screen,struct zink_resource * res,struct zink_bo * bo,uint32_t bo_offset,uint32_t offset,uint32_t size,bool commit,VkSemaphore wait)758 buffer_commit_single(struct zink_screen *screen, struct zink_resource *res, struct zink_bo *bo, uint32_t bo_offset, uint32_t offset, uint32_t size, bool commit, VkSemaphore wait)
759 {
760 VkSemaphore sem = zink_create_semaphore(screen);
761 VkBindSparseInfo sparse = {0};
762 sparse.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
763 sparse.bufferBindCount = res->obj->storage_buffer ? 2 : 1;
764 sparse.waitSemaphoreCount = !!wait;
765 sparse.pWaitSemaphores = &wait;
766 sparse.signalSemaphoreCount = 1;
767 sparse.pSignalSemaphores = &sem;
768
769 VkSparseBufferMemoryBindInfo sparse_bind[2];
770 sparse_bind[0].buffer = res->obj->buffer;
771 sparse_bind[1].buffer = res->obj->storage_buffer;
772 sparse_bind[0].bindCount = 1;
773 sparse_bind[1].bindCount = 1;
774 sparse.pBufferBinds = sparse_bind;
775
776 VkSparseMemoryBind mem_bind;
777 mem_bind.resourceOffset = offset;
778 mem_bind.size = MIN2(res->base.b.width0 - offset, size);
779 mem_bind.memory = commit ? (bo->mem ? bo->mem : bo->u.slab.real->mem) : VK_NULL_HANDLE;
780 mem_bind.memoryOffset = bo_offset * ZINK_SPARSE_BUFFER_PAGE_SIZE + (commit ? (bo->mem ? 0 : bo->offset) : 0);
781 mem_bind.flags = 0;
782 sparse_bind[0].pBinds = &mem_bind;
783 sparse_bind[1].pBinds = &mem_bind;
784
785 VkResult ret = VKSCR(QueueBindSparse)(screen->queue_sparse, 1, &sparse, VK_NULL_HANDLE);
786 if (zink_screen_handle_vkresult(screen, ret))
787 return sem;
788 VKSCR(DestroySemaphore)(screen->dev, sem, NULL);
789 return VK_NULL_HANDLE;
790 }
791
792 static bool
buffer_bo_commit(struct zink_context * ctx,struct zink_resource * res,uint32_t offset,uint32_t size,bool commit,VkSemaphore * sem)793 buffer_bo_commit(struct zink_context *ctx, struct zink_resource *res, uint32_t offset, uint32_t size, bool commit, VkSemaphore *sem)
794 {
795 bool ok = true;
796 struct zink_screen *screen = zink_screen(ctx->base.screen);
797 struct zink_bo *bo = res->obj->bo;
798 assert(offset % ZINK_SPARSE_BUFFER_PAGE_SIZE == 0);
799 assert(offset <= bo->base.base.size);
800 assert(size <= bo->base.base.size - offset);
801 assert(size % ZINK_SPARSE_BUFFER_PAGE_SIZE == 0 || offset + size == res->obj->size);
802
803 struct zink_sparse_commitment *comm = bo->u.sparse.commitments;
804
805 uint32_t va_page = offset / ZINK_SPARSE_BUFFER_PAGE_SIZE;
806 uint32_t end_va_page = va_page + DIV_ROUND_UP(size, ZINK_SPARSE_BUFFER_PAGE_SIZE);
807 VkSemaphore cur_sem = VK_NULL_HANDLE;
808 if (commit) {
809 while (va_page < end_va_page) {
810 uint32_t span_va_page;
811
812 /* Skip pages that are already committed. */
813 if (comm[va_page].backing) {
814 va_page++;
815 continue;
816 }
817
818 /* Determine length of uncommitted span. */
819 span_va_page = va_page;
820 while (va_page < end_va_page && !comm[va_page].backing)
821 va_page++;
822
823 /* Fill the uncommitted span with chunks of backing memory. */
824 while (span_va_page < va_page) {
825 struct zink_sparse_backing *backing;
826 uint32_t backing_start, backing_size;
827
828 backing_size = va_page - span_va_page;
829 backing = sparse_backing_alloc(screen, bo, &backing_start, &backing_size);
830 if (!backing) {
831 ok = false;
832 goto out;
833 }
834 cur_sem = buffer_commit_single(screen, res, backing->bo, backing_start,
835 (uint64_t)span_va_page * ZINK_SPARSE_BUFFER_PAGE_SIZE,
836 (uint64_t)backing_size * ZINK_SPARSE_BUFFER_PAGE_SIZE, true, cur_sem);
837 if (cur_sem) {
838 util_dynarray_append(&ctx->bs->tracked_semaphores, VkSemaphore, cur_sem);
839 } else {
840 ok = sparse_backing_free(screen, bo, backing, backing_start, backing_size);
841 assert(ok && "sufficient memory should already be allocated");
842
843 ok = false;
844 goto out;
845 }
846
847 while (backing_size) {
848 comm[span_va_page].backing = backing;
849 comm[span_va_page].page = backing_start;
850 span_va_page++;
851 backing_start++;
852 backing_size--;
853 }
854 }
855 }
856 } else {
857 bool done = false;
858 uint32_t base_page = va_page;
859 while (va_page < end_va_page) {
860 struct zink_sparse_backing *backing;
861 uint32_t backing_start;
862 uint32_t span_pages;
863
864 /* Skip pages that are already uncommitted. */
865 if (!comm[va_page].backing) {
866 va_page++;
867 continue;
868 }
869
870 if (!done) {
871 cur_sem = buffer_commit_single(screen, res, NULL, 0,
872 (uint64_t)base_page * ZINK_SPARSE_BUFFER_PAGE_SIZE,
873 (uint64_t)(end_va_page - base_page) * ZINK_SPARSE_BUFFER_PAGE_SIZE, false, cur_sem);
874 if (cur_sem) {
875 util_dynarray_append(&ctx->bs->tracked_semaphores, VkSemaphore, cur_sem);
876 } else {
877 ok = false;
878 goto out;
879 }
880 }
881 done = true;
882
883 /* Group contiguous spans of pages. */
884 backing = comm[va_page].backing;
885 backing_start = comm[va_page].page;
886 comm[va_page].backing = NULL;
887
888 span_pages = 1;
889 va_page++;
890
891 while (va_page < end_va_page &&
892 comm[va_page].backing == backing &&
893 comm[va_page].page == backing_start + span_pages) {
894 comm[va_page].backing = NULL;
895 va_page++;
896 span_pages++;
897 }
898
899 track_freed_sparse_bo(ctx, backing);
900 if (!sparse_backing_free(screen, bo, backing, backing_start, span_pages)) {
901 /* Couldn't allocate tracking data structures, so we have to leak */
902 fprintf(stderr, "zink: leaking sparse backing memory\n");
903 ok = false;
904 }
905 }
906 }
907 out:
908 *sem = cur_sem;
909 return ok;
910 }
911
912 static VkSemaphore
texture_commit_single(struct zink_screen * screen,struct zink_resource * res,VkSparseImageMemoryBind * ibind,unsigned num_binds,bool commit,VkSemaphore wait)913 texture_commit_single(struct zink_screen *screen, struct zink_resource *res, VkSparseImageMemoryBind *ibind, unsigned num_binds, bool commit, VkSemaphore wait)
914 {
915 VkSemaphore sem = zink_create_semaphore(screen);
916 VkBindSparseInfo sparse = {0};
917 sparse.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
918 sparse.imageBindCount = 1;
919 sparse.waitSemaphoreCount = !!wait;
920 sparse.pWaitSemaphores = &wait;
921 sparse.signalSemaphoreCount = 1;
922 sparse.pSignalSemaphores = &sem;
923
924 VkSparseImageMemoryBindInfo sparse_ibind;
925 sparse_ibind.image = res->obj->image;
926 sparse_ibind.bindCount = num_binds;
927 sparse_ibind.pBinds = ibind;
928 sparse.pImageBinds = &sparse_ibind;
929
930 VkResult ret = VKSCR(QueueBindSparse)(screen->queue_sparse, 1, &sparse, VK_NULL_HANDLE);
931 if (zink_screen_handle_vkresult(screen, ret))
932 return sem;
933 VKSCR(DestroySemaphore)(screen->dev, sem, NULL);
934 return VK_NULL_HANDLE;
935 }
936
937 static VkSemaphore
texture_commit_miptail(struct zink_screen * screen,struct zink_resource * res,struct zink_bo * bo,uint32_t bo_offset,uint32_t offset,bool commit,VkSemaphore wait)938 texture_commit_miptail(struct zink_screen *screen, struct zink_resource *res, struct zink_bo *bo, uint32_t bo_offset, uint32_t offset, bool commit, VkSemaphore wait)
939 {
940 VkSemaphore sem = zink_create_semaphore(screen);
941 VkBindSparseInfo sparse = {0};
942 sparse.sType = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
943 sparse.imageOpaqueBindCount = 1;
944 sparse.waitSemaphoreCount = !!wait;
945 sparse.pWaitSemaphores = &wait;
946 sparse.signalSemaphoreCount = 1;
947 sparse.pSignalSemaphores = &sem;
948
949 VkSparseImageOpaqueMemoryBindInfo sparse_bind;
950 sparse_bind.image = res->obj->image;
951 sparse_bind.bindCount = 1;
952 sparse.pImageOpaqueBinds = &sparse_bind;
953
954 VkSparseMemoryBind mem_bind;
955 mem_bind.resourceOffset = offset;
956 mem_bind.size = res->sparse.imageMipTailSize;
957 mem_bind.memory = commit ? (bo->mem ? bo->mem : bo->u.slab.real->mem) : VK_NULL_HANDLE;
958 mem_bind.memoryOffset = bo_offset + (commit ? (bo->mem ? 0 : bo->offset) : 0);
959 mem_bind.flags = 0;
960 sparse_bind.pBinds = &mem_bind;
961
962 VkResult ret = VKSCR(QueueBindSparse)(screen->queue_sparse, 1, &sparse, VK_NULL_HANDLE);
963 if (zink_screen_handle_vkresult(screen, ret))
964 return sem;
965 VKSCR(DestroySemaphore)(screen->dev, sem, NULL);
966 return VK_NULL_HANDLE;
967 }
968
969 bool
zink_bo_commit(struct zink_context * ctx,struct zink_resource * res,unsigned level,struct pipe_box * box,bool commit,VkSemaphore * sem)970 zink_bo_commit(struct zink_context *ctx, struct zink_resource *res, unsigned level, struct pipe_box *box, bool commit, VkSemaphore *sem)
971 {
972 bool ok = true;
973 struct zink_screen *screen = zink_screen(ctx->base.screen);
974 struct zink_bo *bo = res->obj->bo;
975 VkSemaphore cur_sem = *sem;
976
977 simple_mtx_lock(&screen->queue_lock);
978 simple_mtx_lock(&bo->lock);
979 if (res->base.b.target == PIPE_BUFFER) {
980 ok = buffer_bo_commit(ctx, res, box->x, box->width, commit, &cur_sem);
981 goto out;
982 }
983
984 int gwidth, gheight, gdepth;
985 gwidth = res->sparse.formatProperties.imageGranularity.width;
986 gheight = res->sparse.formatProperties.imageGranularity.height;
987 gdepth = res->sparse.formatProperties.imageGranularity.depth;
988 assert(gwidth && gheight && gdepth);
989
990 struct zink_sparse_commitment *comm = bo->u.sparse.commitments;
991 bool is_miptail_only = level >= res->sparse.imageMipTailFirstLod;
992 unsigned miptail_clamped_level = level >= res->sparse.imageMipTailFirstLod ? res->sparse.imageMipTailFirstLod : level;
993 VkImageSubresource subresource = { res->aspect, miptail_clamped_level, 0 };
994 unsigned nwidth = is_miptail_only ? 1 : DIV_ROUND_UP(box->width, gwidth);
995 unsigned nheight = is_miptail_only ? 1 : DIV_ROUND_UP(box->height, gheight);
996 unsigned ndepth = is_miptail_only ? 1 : DIV_ROUND_UP(box->depth, gdepth);
997 VkExtent3D lastBlockExtent = {
998 (box->width % gwidth) ? box->width % gwidth : gwidth,
999 (box->height % gheight) ? box->height % gheight : gheight,
1000 (box->depth % gdepth) ? box->depth % gdepth : gdepth
1001 };
1002 #define NUM_BATCHED_BINDS 50
1003 VkSparseImageMemoryBind ibind[NUM_BATCHED_BINDS];
1004 uint32_t backing_start[NUM_BATCHED_BINDS], backing_size[NUM_BATCHED_BINDS];
1005 struct zink_sparse_backing *backing[NUM_BATCHED_BINDS];
1006 unsigned i = 0;
1007 bool commits_pending = false;
1008 uint32_t va_page_offset = 0;
1009 for (unsigned l = 0; l < miptail_clamped_level; l++) {
1010 unsigned mipwidth = DIV_ROUND_UP(u_minify(res->base.b.width0, l), gwidth);
1011 unsigned mipheight = DIV_ROUND_UP(u_minify(res->base.b.height0, l), gheight);
1012 unsigned mipdepth = DIV_ROUND_UP(res->base.b.array_size > 1 ? res->base.b.array_size : u_minify(res->base.b.depth0, l), gdepth);
1013 va_page_offset += mipwidth * mipheight * mipdepth;
1014 }
1015 for (unsigned d = 0; d < ndepth; d++) {
1016 for (unsigned h = 0; h < nheight; h++) {
1017 for (unsigned w = 0; w < nwidth; w++) {
1018 ibind[i].subresource = subresource;
1019 ibind[i].flags = 0;
1020 if (is_miptail_only) {
1021 ibind[i].offset.x = 0;
1022 ibind[i].offset.y = 0;
1023 ibind[i].offset.z = 0;
1024 ibind[i].subresource.arrayLayer = 0;
1025 ibind[i].extent.width = u_minify(res->base.b.width0, miptail_clamped_level);
1026 ibind[i].extent.height = u_minify(res->base.b.height0, miptail_clamped_level);
1027 ibind[i].extent.depth = u_minify(res->base.b.depth0, miptail_clamped_level);
1028 } else {
1029 // Offset
1030 ibind[i].offset.x = w * gwidth;
1031 ibind[i].offset.y = h * gheight;
1032 if (res->base.b.array_size > 1) {
1033 ibind[i].subresource.arrayLayer = d * gdepth;
1034 ibind[i].offset.z = 0;
1035 } else {
1036 ibind[i].offset.z = d * gdepth;
1037 }
1038 // Size of the page
1039 ibind[i].extent.width = (w == nwidth - 1) ? lastBlockExtent.width : gwidth;
1040 ibind[i].extent.height = (h == nheight - 1) ? lastBlockExtent.height : gheight;
1041 ibind[i].extent.depth = (d == ndepth - 1 && res->base.b.target != PIPE_TEXTURE_CUBE) ? lastBlockExtent.depth : gdepth;
1042 }
1043 uint32_t va_page = va_page_offset;
1044 /* single miptail binds use the base miptail page */
1045 if (!is_miptail_only) {
1046 va_page +=
1047 (d + (box->z / gdepth)) * (u_minify(res->base.b.width0, miptail_clamped_level) / gwidth) * (u_minify(res->base.b.height0, miptail_clamped_level) / gheight) +
1048 (h + (box->y / gheight)) * (u_minify(res->base.b.width0, miptail_clamped_level) / gwidth) +
1049 (w + (box->x / gwidth));
1050 }
1051
1052 uint32_t end_va_page = va_page + 1;
1053
1054 if (commit) {
1055 while (va_page < end_va_page) {
1056 uint32_t span_va_page;
1057
1058 /* Skip pages that are already committed. */
1059 if (comm[va_page].backing) {
1060 va_page++;
1061 continue;
1062 }
1063
1064 /* Determine length of uncommitted span. */
1065 span_va_page = va_page;
1066 while (va_page < end_va_page && !comm[va_page].backing)
1067 va_page++;
1068
1069 /* Fill the uncommitted span with chunks of backing memory. */
1070 while (span_va_page < va_page) {
1071 backing_size[i] = va_page - span_va_page;
1072 backing[i] = sparse_backing_alloc(screen, bo, &backing_start[i], &backing_size[i]);
1073 if (!backing[i]) {
1074 ok = false;
1075 goto out;
1076 }
1077 if (level >= res->sparse.imageMipTailFirstLod) {
1078 uint32_t offset = res->sparse.imageMipTailOffset;
1079 cur_sem = texture_commit_miptail(screen, res, backing[i]->bo, backing_start[i], offset, commit, cur_sem);
1080 if (cur_sem) {
1081 util_dynarray_append(&ctx->bs->tracked_semaphores, VkSemaphore, cur_sem);
1082 res->obj->miptail_commits++;
1083 } else {
1084 ok = false;
1085 }
1086 goto out;
1087 } else {
1088 ibind[i].memory = backing[i]->bo->mem ? backing[i]->bo->mem : backing[i]->bo->u.slab.real->mem;
1089 ibind[i].memoryOffset = backing_start[i] * ZINK_SPARSE_BUFFER_PAGE_SIZE +
1090 (backing[i]->bo->mem ? 0 : backing[i]->bo->offset);
1091 commits_pending = true;
1092 }
1093
1094 while (backing_size[i]) {
1095 comm[span_va_page].backing = backing[i];
1096 comm[span_va_page].page = backing_start[i];
1097 span_va_page++;
1098 backing_start[i]++;
1099 backing_size[i]--;
1100 }
1101 i++;
1102 }
1103 }
1104 } else {
1105 ibind[i].memory = VK_NULL_HANDLE;
1106 ibind[i].memoryOffset = 0;
1107
1108 while (va_page < end_va_page) {
1109 /* Skip pages that are already uncommitted. */
1110 if (!comm[va_page].backing) {
1111 va_page++;
1112 continue;
1113 }
1114
1115 /* Group contiguous spans of pages. */
1116 backing[i] = comm[va_page].backing;
1117 backing_start[i] = comm[va_page].page;
1118 comm[va_page].backing = NULL;
1119
1120 backing_size[i] = 1;
1121 va_page++;
1122
1123 while (va_page < end_va_page &&
1124 comm[va_page].backing == backing[i] &&
1125 comm[va_page].page == backing_start[i] + backing_size[i]) {
1126 comm[va_page].backing = NULL;
1127 va_page++;
1128 backing_size[i]++;
1129 }
1130 if (level >= res->sparse.imageMipTailFirstLod) {
1131 uint32_t offset = res->sparse.imageMipTailOffset;
1132 assert(res->obj->miptail_commits);
1133 res->obj->miptail_commits--;
1134 if (!res->obj->miptail_commits) {
1135 cur_sem = texture_commit_miptail(screen, res, NULL, 0, offset, commit, cur_sem);
1136 if (cur_sem)
1137 util_dynarray_append(&ctx->bs->tracked_semaphores, VkSemaphore, cur_sem);
1138 else
1139 ok = false;
1140 ok = sparse_backing_free(screen, backing[i]->bo, backing[i], backing_start[i], backing_size[i]);
1141 if (!ok) {
1142 /* Couldn't allocate tracking data structures, so we have to leak */
1143 fprintf(stderr, "zink: leaking sparse backing memory\n");
1144 }
1145 }
1146 goto out;
1147 } else {
1148 commits_pending = true;
1149 }
1150 i++;
1151 }
1152 }
1153 if (i == ARRAY_SIZE(ibind)) {
1154 cur_sem = texture_commit_single(screen, res, ibind, ARRAY_SIZE(ibind), commit, cur_sem);
1155 if (cur_sem) {
1156 util_dynarray_append(&ctx->bs->tracked_semaphores, VkSemaphore, cur_sem);
1157 } else {
1158 for (unsigned s = 0; s < i; s++) {
1159 ok = sparse_backing_free(screen, backing[s]->bo, backing[s], backing_start[s], backing_size[s]);
1160 if (!ok) {
1161 /* Couldn't allocate tracking data structures, so we have to leak */
1162 fprintf(stderr, "zink: leaking sparse backing memory\n");
1163 }
1164 }
1165 ok = false;
1166 goto out;
1167 }
1168 commits_pending = false;
1169 i = 0;
1170 }
1171 }
1172 }
1173 }
1174 if (commits_pending) {
1175 cur_sem = texture_commit_single(screen, res, ibind, i, commit, cur_sem);
1176 if (cur_sem) {
1177 util_dynarray_append(&ctx->bs->tracked_semaphores, VkSemaphore, cur_sem);
1178 } else {
1179 for (unsigned s = 0; s < i; s++) {
1180 ok = sparse_backing_free(screen, backing[s]->bo, backing[s], backing_start[s], backing_size[s]);
1181 if (!ok) {
1182 /* Couldn't allocate tracking data structures, so we have to leak */
1183 fprintf(stderr, "zink: leaking sparse backing memory\n");
1184 }
1185 }
1186 ok = false;
1187 }
1188 }
1189 out:
1190
1191 simple_mtx_unlock(&bo->lock);
1192 simple_mtx_unlock(&screen->queue_lock);
1193 *sem = cur_sem;
1194 return ok;
1195 }
1196
1197 bool
zink_bo_get_kms_handle(struct zink_screen * screen,struct zink_bo * bo,int fd,uint32_t * handle)1198 zink_bo_get_kms_handle(struct zink_screen *screen, struct zink_bo *bo, int fd, uint32_t *handle)
1199 {
1200 #ifdef ZINK_USE_DMABUF
1201 assert(bo->mem && !bo->u.real.use_reusable_pool);
1202 simple_mtx_lock(&bo->u.real.export_lock);
1203 list_for_each_entry(struct bo_export, export, &bo->u.real.exports, link) {
1204 if (export->drm_fd == fd) {
1205 simple_mtx_unlock(&bo->u.real.export_lock);
1206 *handle = export->gem_handle;
1207 return true;
1208 }
1209 }
1210 struct bo_export *export = CALLOC_STRUCT(bo_export);
1211 if (!export) {
1212 simple_mtx_unlock(&bo->u.real.export_lock);
1213 return false;
1214 }
1215 bool success = drmPrimeFDToHandle(screen->drm_fd, fd, handle) == 0;
1216 if (success) {
1217 list_addtail(&export->link, &bo->u.real.exports);
1218 export->gem_handle = *handle;
1219 export->drm_fd = screen->drm_fd;
1220 } else {
1221 mesa_loge("zink: failed drmPrimeFDToHandle %s", strerror(errno));
1222 FREE(export);
1223 }
1224 simple_mtx_unlock(&bo->u.real.export_lock);
1225 return success;
1226 #else
1227 return false;
1228 #endif
1229 }
1230
1231 static const struct pb_vtbl bo_slab_vtbl = {
1232 /* Cast to void* because one of the function parameters is a struct pointer instead of void*. */
1233 (void*)bo_slab_destroy
1234 /* other functions are never called */
1235 };
1236
1237 static struct pb_slab *
bo_slab_alloc(void * priv,unsigned mem_type_idx,unsigned entry_size,unsigned group_index,bool encrypted)1238 bo_slab_alloc(void *priv, unsigned mem_type_idx, unsigned entry_size, unsigned group_index, bool encrypted)
1239 {
1240 struct zink_screen *screen = priv;
1241 unsigned slab_size = 0;
1242 struct zink_slab *slab = CALLOC_STRUCT(zink_slab);
1243
1244 if (!slab)
1245 return NULL;
1246
1247 //struct pb_slabs *slabs = ((flags & RADEON_FLAG_ENCRYPTED) && screen->info.has_tmz_support) ?
1248 //screen->bo_slabs_encrypted : screen->bo_slabs;
1249 struct pb_slabs *slabs = screen->pb.bo_slabs;
1250
1251 /* Determine the slab buffer size. */
1252 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
1253 unsigned max_entry_size = 1 << (slabs[i].min_order + slabs[i].num_orders - 1);
1254
1255 if (entry_size <= max_entry_size) {
1256 /* The slab size is twice the size of the largest possible entry. */
1257 slab_size = max_entry_size * 2;
1258
1259 if (!util_is_power_of_two_nonzero(entry_size)) {
1260 assert(util_is_power_of_two_nonzero(entry_size * 4 / 3));
1261
1262 /* If the entry size is 3/4 of a power of two, we would waste space and not gain
1263 * anything if we allocated only twice the power of two for the backing buffer:
1264 * 2 * 3/4 = 1.5 usable with buffer size 2
1265 *
1266 * Allocating 5 times the entry size leads us to the next power of two and results
1267 * in a much better memory utilization:
1268 * 5 * 3/4 = 3.75 usable with buffer size 4
1269 */
1270 if (entry_size * 5 > slab_size)
1271 slab_size = util_next_power_of_two(entry_size * 5);
1272 }
1273
1274 break;
1275 }
1276 }
1277 assert(slab_size != 0);
1278
1279 slab->buffer = zink_bo(zink_bo_create(screen, slab_size, slab_size, zink_heap_from_domain_flags(screen->info.mem_props.memoryTypes[mem_type_idx].propertyFlags, 0),
1280 0, mem_type_idx, NULL));
1281 if (!slab->buffer)
1282 goto fail;
1283
1284 slab_size = slab->buffer->base.base.size;
1285
1286 slab->base.num_entries = slab_size / entry_size;
1287 slab->base.num_free = slab->base.num_entries;
1288 slab->base.group_index = group_index;
1289 slab->base.entry_size = entry_size;
1290 slab->entries = CALLOC(slab->base.num_entries, sizeof(*slab->entries));
1291 if (!slab->entries)
1292 goto fail_buffer;
1293
1294 list_inithead(&slab->base.free);
1295
1296 for (unsigned i = 0; i < slab->base.num_entries; ++i) {
1297 struct zink_bo *bo = &slab->entries[i];
1298
1299 simple_mtx_init(&bo->lock, mtx_plain);
1300 bo->base.base.alignment_log2 = util_logbase2(get_slab_entry_alignment(screen, entry_size));
1301 bo->base.base.size = entry_size;
1302 bo->base.vtbl = &bo_slab_vtbl;
1303 bo->offset = slab->buffer->offset + i * entry_size;
1304 bo->u.slab.entry.slab = &slab->base;
1305
1306 if (slab->buffer->mem) {
1307 /* The slab is not suballocated. */
1308 bo->u.slab.real = slab->buffer;
1309 } else {
1310 /* The slab is allocated out of a bigger slab. */
1311 bo->u.slab.real = slab->buffer->u.slab.real;
1312 assert(bo->u.slab.real->mem);
1313 }
1314 bo->base.base.placement = bo->u.slab.real->base.base.placement;
1315
1316 list_addtail(&bo->u.slab.entry.head, &slab->base.free);
1317 }
1318
1319 /* Wasted alignment due to slabs with 3/4 allocations being aligned to a power of two. */
1320 assert(slab->base.num_entries * entry_size <= slab_size);
1321
1322 return &slab->base;
1323
1324 fail_buffer:
1325 zink_bo_unref(screen, slab->buffer);
1326 fail:
1327 FREE(slab);
1328 return NULL;
1329 }
1330
1331 static struct pb_slab *
bo_slab_alloc_normal(void * priv,unsigned mem_type_idx,unsigned entry_size,unsigned group_index)1332 bo_slab_alloc_normal(void *priv, unsigned mem_type_idx, unsigned entry_size, unsigned group_index)
1333 {
1334 return bo_slab_alloc(priv, mem_type_idx, entry_size, group_index, false);
1335 }
1336
1337 bool
zink_bo_init(struct zink_screen * screen)1338 zink_bo_init(struct zink_screen *screen)
1339 {
1340 uint64_t total_mem = 0;
1341 for (uint32_t i = 0; i < screen->info.mem_props.memoryHeapCount; ++i)
1342 total_mem += screen->info.mem_props.memoryHeaps[i].size;
1343 /* Create managers. */
1344 pb_cache_init(&screen->pb.bo_cache, screen->info.mem_props.memoryTypeCount,
1345 500000, 2.0f, 0,
1346 total_mem / 8, offsetof(struct zink_bo, cache_entry), screen,
1347 (void*)bo_destroy, (void*)bo_can_reclaim);
1348
1349 unsigned min_slab_order = MIN_SLAB_ORDER; /* 256 bytes */
1350 unsigned max_slab_order = 20; /* 1 MB (slab size = 2 MB) */
1351 unsigned num_slab_orders_per_allocator = (max_slab_order - min_slab_order) /
1352 NUM_SLAB_ALLOCATORS;
1353
1354 /* Divide the size order range among slab managers. */
1355 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
1356 unsigned min_order = min_slab_order;
1357 unsigned max_order = MIN2(min_order + num_slab_orders_per_allocator,
1358 max_slab_order);
1359
1360 if (!pb_slabs_init(&screen->pb.bo_slabs[i],
1361 min_order, max_order,
1362 screen->info.mem_props.memoryTypeCount, true,
1363 screen,
1364 bo_can_reclaim_slab,
1365 bo_slab_alloc_normal,
1366 (void*)bo_slab_free)) {
1367 return false;
1368 }
1369 min_slab_order = max_order + 1;
1370 }
1371 screen->pb.min_alloc_size = 1 << screen->pb.bo_slabs[0].min_order;
1372 return true;
1373 }
1374
1375 void
zink_bo_deinit(struct zink_screen * screen)1376 zink_bo_deinit(struct zink_screen *screen)
1377 {
1378 for (unsigned i = 0; i < NUM_SLAB_ALLOCATORS; i++) {
1379 if (screen->pb.bo_slabs[i].groups)
1380 pb_slabs_deinit(&screen->pb.bo_slabs[i]);
1381 }
1382 pb_cache_deinit(&screen->pb.bo_cache);
1383 }
1384