1 /*
2 * Copyright © 2014-2017 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include <errno.h>
25 #include <err.h>
26 #include <sys/mman.h>
27 #include <fcntl.h>
28 #include <xf86drm.h>
29 #include <xf86drmMode.h>
30
31 #include "util/u_hash_table.h"
32 #include "util/u_memory.h"
33 #include "util/ralloc.h"
34
35 #include "v3d_context.h"
36 #include "v3d_screen.h"
37
38 static bool dump_stats = false;
39
40 static void
41 v3d_bo_cache_free_all(struct v3d_bo_cache *cache);
42
43 static void
v3d_bo_dump_stats(struct v3d_screen * screen)44 v3d_bo_dump_stats(struct v3d_screen *screen)
45 {
46 struct v3d_bo_cache *cache = &screen->bo_cache;
47
48 uint32_t cache_count = 0;
49 uint32_t cache_size = 0;
50 list_for_each_entry(struct v3d_bo, bo, &cache->time_list, time_list) {
51 cache_count++;
52 cache_size += bo->size;
53 }
54
55 fprintf(stderr, " BOs allocated: %d\n", screen->bo_count);
56 fprintf(stderr, " BOs size: %dkb\n", screen->bo_size / 1024);
57 fprintf(stderr, " BOs cached: %d\n", cache_count);
58 fprintf(stderr, " BOs cached size: %dkb\n", cache_size / 1024);
59
60 if (!list_is_empty(&cache->time_list)) {
61 struct v3d_bo *first = list_first_entry(&cache->time_list,
62 struct v3d_bo,
63 time_list);
64 struct v3d_bo *last = list_last_entry(&cache->time_list,
65 struct v3d_bo,
66 time_list);
67
68 fprintf(stderr, " oldest cache time: %ld\n",
69 (long)first->free_time);
70 fprintf(stderr, " newest cache time: %ld\n",
71 (long)last->free_time);
72
73 struct timespec time;
74 clock_gettime(CLOCK_MONOTONIC, &time);
75 fprintf(stderr, " now: %jd\n",
76 (intmax_t)time.tv_sec);
77 }
78 }
79
80 static void
v3d_bo_remove_from_cache(struct v3d_bo_cache * cache,struct v3d_bo * bo)81 v3d_bo_remove_from_cache(struct v3d_bo_cache *cache, struct v3d_bo *bo)
82 {
83 list_del(&bo->time_list);
84 list_del(&bo->size_list);
85 }
86
87 static struct v3d_bo *
v3d_bo_from_cache(struct v3d_screen * screen,uint32_t size,const char * name)88 v3d_bo_from_cache(struct v3d_screen *screen, uint32_t size, const char *name)
89 {
90 struct v3d_bo_cache *cache = &screen->bo_cache;
91 uint32_t page_index = size / 4096 - 1;
92
93 if (cache->size_list_size <= page_index)
94 return NULL;
95
96 struct v3d_bo *bo = NULL;
97 mtx_lock(&cache->lock);
98 if (!list_is_empty(&cache->size_list[page_index])) {
99 bo = list_first_entry(&cache->size_list[page_index],
100 struct v3d_bo, size_list);
101
102 /* Check that the BO has gone idle. If not, then we want to
103 * allocate something new instead, since we assume that the
104 * user will proceed to CPU map it and fill it with stuff.
105 */
106 if (!v3d_bo_wait(bo, 0, NULL)) {
107 mtx_unlock(&cache->lock);
108 return NULL;
109 }
110
111 pipe_reference_init(&bo->reference, 1);
112 v3d_bo_remove_from_cache(cache, bo);
113
114 bo->name = name;
115 }
116 mtx_unlock(&cache->lock);
117 return bo;
118 }
119
120 struct v3d_bo *
v3d_bo_alloc(struct v3d_screen * screen,uint32_t size,const char * name)121 v3d_bo_alloc(struct v3d_screen *screen, uint32_t size, const char *name)
122 {
123 struct v3d_bo *bo;
124 int ret;
125
126 /* The CLIF dumping requires that there is no whitespace in the name.
127 */
128 assert(!strchr(name, ' '));
129
130 size = align(size, 4096);
131
132 bo = v3d_bo_from_cache(screen, size, name);
133 if (bo) {
134 if (dump_stats) {
135 fprintf(stderr, "Allocated %s %dkb from cache:\n",
136 name, size / 1024);
137 v3d_bo_dump_stats(screen);
138 }
139 return bo;
140 }
141
142 bo = CALLOC_STRUCT(v3d_bo);
143 if (!bo)
144 return NULL;
145
146 pipe_reference_init(&bo->reference, 1);
147 bo->screen = screen;
148 bo->size = size;
149 bo->name = name;
150 bo->private = true;
151
152 retry:
153 ;
154
155 bool cleared_and_retried = false;
156 struct drm_v3d_create_bo create = {
157 .size = size
158 };
159
160 ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_CREATE_BO, &create);
161 bo->handle = create.handle;
162 bo->offset = create.offset;
163
164 if (ret != 0) {
165 if (!list_is_empty(&screen->bo_cache.time_list) &&
166 !cleared_and_retried) {
167 cleared_and_retried = true;
168 v3d_bo_cache_free_all(&screen->bo_cache);
169 goto retry;
170 }
171
172 free(bo);
173 return NULL;
174 }
175
176 screen->bo_count++;
177 screen->bo_size += bo->size;
178 if (dump_stats) {
179 fprintf(stderr, "Allocated %s %dkb:\n", name, size / 1024);
180 v3d_bo_dump_stats(screen);
181 }
182
183 return bo;
184 }
185
186 void
v3d_bo_last_unreference(struct v3d_bo * bo)187 v3d_bo_last_unreference(struct v3d_bo *bo)
188 {
189 struct v3d_screen *screen = bo->screen;
190
191 struct timespec time;
192 clock_gettime(CLOCK_MONOTONIC, &time);
193 mtx_lock(&screen->bo_cache.lock);
194 v3d_bo_last_unreference_locked_timed(bo, time.tv_sec);
195 mtx_unlock(&screen->bo_cache.lock);
196 }
197
198 static void
v3d_bo_free(struct v3d_bo * bo)199 v3d_bo_free(struct v3d_bo *bo)
200 {
201 struct v3d_screen *screen = bo->screen;
202
203 if (bo->map) {
204 #if USE_V3D_SIMULATOR
205 if (bo->name &&
206 strcmp(bo->name, "winsys") == 0) {
207 free(bo->map);
208 } else
209 #endif
210 {
211 munmap(bo->map, bo->size);
212 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
213 }
214 }
215
216 struct drm_gem_close c;
217 memset(&c, 0, sizeof(c));
218 c.handle = bo->handle;
219 int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_CLOSE, &c);
220 if (ret != 0)
221 fprintf(stderr, "close object %d: %s\n", bo->handle, strerror(errno));
222
223 screen->bo_count--;
224 screen->bo_size -= bo->size;
225
226 if (dump_stats) {
227 fprintf(stderr, "Freed %s%s%dkb:\n",
228 bo->name ? bo->name : "",
229 bo->name ? " " : "",
230 bo->size / 1024);
231 v3d_bo_dump_stats(screen);
232 }
233
234 free(bo);
235 }
236
237 static void
free_stale_bos(struct v3d_screen * screen,time_t time)238 free_stale_bos(struct v3d_screen *screen, time_t time)
239 {
240 struct v3d_bo_cache *cache = &screen->bo_cache;
241 bool freed_any = false;
242
243 list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
244 time_list) {
245 /* If it's more than a second old, free it. */
246 if (time - bo->free_time > 2) {
247 if (dump_stats && !freed_any) {
248 fprintf(stderr, "Freeing stale BOs:\n");
249 v3d_bo_dump_stats(screen);
250 freed_any = true;
251 }
252 v3d_bo_remove_from_cache(cache, bo);
253 v3d_bo_free(bo);
254 } else {
255 break;
256 }
257 }
258
259 if (dump_stats && freed_any) {
260 fprintf(stderr, "Freed stale BOs:\n");
261 v3d_bo_dump_stats(screen);
262 }
263 }
264
265 static void
v3d_bo_cache_free_all(struct v3d_bo_cache * cache)266 v3d_bo_cache_free_all(struct v3d_bo_cache *cache)
267 {
268 mtx_lock(&cache->lock);
269 list_for_each_entry_safe(struct v3d_bo, bo, &cache->time_list,
270 time_list) {
271 v3d_bo_remove_from_cache(cache, bo);
272 v3d_bo_free(bo);
273 }
274 mtx_unlock(&cache->lock);
275 }
276
277 void
v3d_bo_last_unreference_locked_timed(struct v3d_bo * bo,time_t time)278 v3d_bo_last_unreference_locked_timed(struct v3d_bo *bo, time_t time)
279 {
280 struct v3d_screen *screen = bo->screen;
281 struct v3d_bo_cache *cache = &screen->bo_cache;
282 uint32_t page_index = bo->size / 4096 - 1;
283
284 if (!bo->private) {
285 v3d_bo_free(bo);
286 return;
287 }
288
289 if (cache->size_list_size <= page_index) {
290 struct list_head *new_list =
291 ralloc_array(screen, struct list_head, page_index + 1);
292
293 /* Move old list contents over (since the array has moved, and
294 * therefore the pointers to the list heads have to change).
295 */
296 for (int i = 0; i < cache->size_list_size; i++) {
297 struct list_head *old_head = &cache->size_list[i];
298 if (list_is_empty(old_head))
299 list_inithead(&new_list[i]);
300 else {
301 new_list[i].next = old_head->next;
302 new_list[i].prev = old_head->prev;
303 new_list[i].next->prev = &new_list[i];
304 new_list[i].prev->next = &new_list[i];
305 }
306 }
307 for (int i = cache->size_list_size; i < page_index + 1; i++)
308 list_inithead(&new_list[i]);
309
310 cache->size_list = new_list;
311 cache->size_list_size = page_index + 1;
312 }
313
314 bo->free_time = time;
315 list_addtail(&bo->size_list, &cache->size_list[page_index]);
316 list_addtail(&bo->time_list, &cache->time_list);
317 if (dump_stats) {
318 fprintf(stderr, "Freed %s %dkb to cache:\n",
319 bo->name, bo->size / 1024);
320 v3d_bo_dump_stats(screen);
321 }
322 bo->name = NULL;
323
324 free_stale_bos(screen, time);
325 }
326
327 static struct v3d_bo *
v3d_bo_open_handle(struct v3d_screen * screen,uint32_t handle,uint32_t size)328 v3d_bo_open_handle(struct v3d_screen *screen,
329 uint32_t handle, uint32_t size)
330 {
331 struct v3d_bo *bo;
332
333 /* Note: the caller is responsible for locking screen->bo_handles_mutex.
334 * This allows the lock to cover the actual BO import, avoiding a race.
335 */
336
337 assert(size);
338
339 bo = util_hash_table_get(screen->bo_handles, (void*)(uintptr_t)handle);
340 if (bo) {
341 pipe_reference(NULL, &bo->reference);
342 goto done;
343 }
344
345 bo = CALLOC_STRUCT(v3d_bo);
346 pipe_reference_init(&bo->reference, 1);
347 bo->screen = screen;
348 bo->handle = handle;
349 bo->size = size;
350 bo->name = "winsys";
351 bo->private = false;
352
353 #if USE_V3D_SIMULATOR
354 v3d_simulator_open_from_handle(screen->fd, bo->handle, bo->size);
355 bo->map = malloc(bo->size);
356 #endif
357
358 struct drm_v3d_get_bo_offset get = {
359 .handle = handle,
360 };
361 int ret = v3d_ioctl(screen->fd, DRM_IOCTL_V3D_GET_BO_OFFSET, &get);
362 if (ret) {
363 fprintf(stderr, "Failed to get BO offset: %s\n",
364 strerror(errno));
365 free(bo->map);
366 free(bo);
367 bo = NULL;
368 goto done;
369 }
370 bo->offset = get.offset;
371 assert(bo->offset != 0);
372
373 _mesa_hash_table_insert(screen->bo_handles, (void *)(uintptr_t)handle, bo);
374
375 screen->bo_count++;
376 screen->bo_size += bo->size;
377
378 done:
379 mtx_unlock(&screen->bo_handles_mutex);
380 return bo;
381 }
382
383 struct v3d_bo *
v3d_bo_open_name(struct v3d_screen * screen,uint32_t name)384 v3d_bo_open_name(struct v3d_screen *screen, uint32_t name)
385 {
386 struct drm_gem_open o = {
387 .name = name
388 };
389 mtx_lock(&screen->bo_handles_mutex);
390
391 int ret = v3d_ioctl(screen->fd, DRM_IOCTL_GEM_OPEN, &o);
392 if (ret) {
393 fprintf(stderr, "Failed to open bo %d: %s\n",
394 name, strerror(errno));
395 mtx_unlock(&screen->bo_handles_mutex);
396 return NULL;
397 }
398
399 return v3d_bo_open_handle(screen, o.handle, o.size);
400 }
401
402 struct v3d_bo *
v3d_bo_open_dmabuf(struct v3d_screen * screen,int fd)403 v3d_bo_open_dmabuf(struct v3d_screen *screen, int fd)
404 {
405 uint32_t handle;
406
407 mtx_lock(&screen->bo_handles_mutex);
408
409 int ret = drmPrimeFDToHandle(screen->fd, fd, &handle);
410 int size;
411 if (ret) {
412 fprintf(stderr, "Failed to get v3d handle for dmabuf %d\n", fd);
413 mtx_unlock(&screen->bo_handles_mutex);
414 return NULL;
415 }
416
417 /* Determine the size of the bo we were handed. */
418 size = lseek(fd, 0, SEEK_END);
419 if (size == -1) {
420 fprintf(stderr, "Couldn't get size of dmabuf fd %d.\n", fd);
421 mtx_unlock(&screen->bo_handles_mutex);
422 return NULL;
423 }
424
425 return v3d_bo_open_handle(screen, handle, size);
426 }
427
428 int
v3d_bo_get_dmabuf(struct v3d_bo * bo)429 v3d_bo_get_dmabuf(struct v3d_bo *bo)
430 {
431 int fd;
432 int ret = drmPrimeHandleToFD(bo->screen->fd, bo->handle,
433 O_CLOEXEC, &fd);
434 if (ret != 0) {
435 fprintf(stderr, "Failed to export gem bo %d to dmabuf\n",
436 bo->handle);
437 return -1;
438 }
439
440 mtx_lock(&bo->screen->bo_handles_mutex);
441 bo->private = false;
442 _mesa_hash_table_insert(bo->screen->bo_handles, (void *)(uintptr_t)bo->handle, bo);
443 mtx_unlock(&bo->screen->bo_handles_mutex);
444
445 return fd;
446 }
447
448 bool
v3d_bo_flink(struct v3d_bo * bo,uint32_t * name)449 v3d_bo_flink(struct v3d_bo *bo, uint32_t *name)
450 {
451 struct drm_gem_flink flink = {
452 .handle = bo->handle,
453 };
454 int ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_GEM_FLINK, &flink);
455 if (ret) {
456 fprintf(stderr, "Failed to flink bo %d: %s\n",
457 bo->handle, strerror(errno));
458 free(bo);
459 return false;
460 }
461
462 bo->private = false;
463 *name = flink.name;
464
465 return true;
466 }
467
v3d_wait_bo_ioctl(int fd,uint32_t handle,uint64_t timeout_ns)468 static int v3d_wait_bo_ioctl(int fd, uint32_t handle, uint64_t timeout_ns)
469 {
470 struct drm_v3d_wait_bo wait = {
471 .handle = handle,
472 .timeout_ns = timeout_ns,
473 };
474 int ret = v3d_ioctl(fd, DRM_IOCTL_V3D_WAIT_BO, &wait);
475 if (ret == -1)
476 return -errno;
477 else
478 return 0;
479
480 }
481
482 bool
v3d_bo_wait(struct v3d_bo * bo,uint64_t timeout_ns,const char * reason)483 v3d_bo_wait(struct v3d_bo *bo, uint64_t timeout_ns, const char *reason)
484 {
485 struct v3d_screen *screen = bo->screen;
486
487 if (V3D_DBG(PERF) && timeout_ns && reason) {
488 if (v3d_wait_bo_ioctl(screen->fd, bo->handle, 0) == -ETIME) {
489 fprintf(stderr, "Blocking on %s BO for %s\n",
490 bo->name, reason);
491 }
492 }
493
494 int ret = v3d_wait_bo_ioctl(screen->fd, bo->handle, timeout_ns);
495 if (ret) {
496 if (ret != -ETIME) {
497 fprintf(stderr, "wait failed: %d\n", ret);
498 abort();
499 }
500
501 return false;
502 }
503
504 return true;
505 }
506
507 void *
v3d_bo_map_unsynchronized(struct v3d_bo * bo)508 v3d_bo_map_unsynchronized(struct v3d_bo *bo)
509 {
510 uint64_t offset;
511 int ret;
512
513 if (bo->map)
514 return bo->map;
515
516 struct drm_v3d_mmap_bo map;
517 memset(&map, 0, sizeof(map));
518 map.handle = bo->handle;
519 ret = v3d_ioctl(bo->screen->fd, DRM_IOCTL_V3D_MMAP_BO, &map);
520 offset = map.offset;
521 if (ret != 0) {
522 fprintf(stderr, "map ioctl failure\n");
523 abort();
524 }
525
526 bo->map = mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
527 bo->screen->fd, offset);
528 if (bo->map == MAP_FAILED) {
529 fprintf(stderr, "mmap of bo %d (offset 0x%016llx, size %d) failed\n",
530 bo->handle, (long long)offset, bo->size);
531 abort();
532 }
533 VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, false));
534
535 return bo->map;
536 }
537
538 void *
v3d_bo_map(struct v3d_bo * bo)539 v3d_bo_map(struct v3d_bo *bo)
540 {
541 void *map = v3d_bo_map_unsynchronized(bo);
542
543 bool ok = v3d_bo_wait(bo, OS_TIMEOUT_INFINITE, "bo map");
544 if (!ok) {
545 fprintf(stderr, "BO wait for map failed\n");
546 abort();
547 }
548
549 return map;
550 }
551
552 void
v3d_bufmgr_destroy(struct pipe_screen * pscreen)553 v3d_bufmgr_destroy(struct pipe_screen *pscreen)
554 {
555 struct v3d_screen *screen = v3d_screen(pscreen);
556 struct v3d_bo_cache *cache = &screen->bo_cache;
557
558 v3d_bo_cache_free_all(cache);
559
560 if (dump_stats) {
561 fprintf(stderr, "BO stats after screen destroy:\n");
562 v3d_bo_dump_stats(screen);
563 }
564 }
565