1 /**************************************************************************
2 *
3 * Copyright © 2007 Red Hat Inc.
4 * Copyright © 2007-2021 Intel Corporation
5 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
26 * of the Software.
27 *
28 *
29 **************************************************************************/
30 /*
31 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
32 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
33 * Eric Anholt <[email protected]>
34 * Dave Airlie <[email protected]>
35 */
36
37 #ifdef HAVE_CONFIG_H
38 #include "config.h"
39 #endif
40
41 #include "xf86drm.h"
42 #include "xf86atomic.h"
43 #include <fcntl.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h>
47 #include <unistd.h>
48 #include <assert.h>
49 #include <pthread.h>
50 #include <sys/ioctl.h>
51 #include <sys/stat.h>
52 #include <sys/types.h>
53 #include <stdbool.h>
54
55 #include "errno.h"
56 #ifndef ETIME
57 #define ETIME ETIMEDOUT
58 #endif
59 #include "libdrm_macros.h"
60 #include "libdrm_lists.h"
61 #include "mos_bufmgr.h"
62 #include "mos_bufmgr_priv.h"
63 #include "string.h"
64
65 #include "i915_drm.h"
66 #include "mos_vma.h"
67
68 #ifdef HAVE_VALGRIND
69 #include <valgrind.h>
70 #include <memcheck.h>
71 #define VG(x) x
72 #else
73 #define VG(x)
74 #endif
75
76 #define memclear(s) memset(&s, 0, sizeof(s))
77 #define MOS_DBG(...) do { \
78 if (bufmgr_gem->bufmgr.debug) \
79 fprintf(stderr, __VA_ARGS__); \
80 } while (0)
81
82 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
83 #define MAX2(A, B) ((A) > (B) ? (A) : (B))
84
85 /**
86 * upper_32_bits - return bits 32-63 of a number
87 * @n: the number we're accessing
88 *
89 * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress
90 * the "right shift count >= width of type" warning when that quantity is
91 * 32-bits.
92 */
93 #define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))
94
95 /**
96 * lower_32_bits - return bits 0-31 of a number
97 * @n: the number we're accessing
98 */
99 #define lower_32_bits(n) ((__u32)(n))
100
101 #define PCI_CHIP_I915_G 0x2582
102 #define PCI_CHIP_E7221_G 0x258A
103 #define PCI_CHIP_I915_GM 0x2592
104
105 #define IS_915(devid) ((devid) == PCI_CHIP_I915_G || \
106 (devid) == PCI_CHIP_E7221_G || \
107 (devid) == PCI_CHIP_I915_GM)
108
109 struct mos_gem_bo_bucket {
110 drmMMListHead head;
111 unsigned long size;
112 };
113
114 struct mos_bufmgr_gem {
115 struct mos_bufmgr bufmgr;
116
117 atomic_t refcount;
118
119 int fd;
120
121 int max_relocs;
122
123 pthread_mutex_t lock;
124
125 struct drm_i915_gem_exec_object *exec_objects;
126 struct drm_i915_gem_exec_object2 *exec2_objects;
127 struct mos_linux_bo **exec_bos;
128 int exec_size;
129 int exec_count;
130
131 /** Array of lists of cached gem objects of power-of-two sizes */
132 struct mos_gem_bo_bucket cache_bucket[14 * 4];
133 int num_buckets;
134 time_t time;
135
136 drmMMListHead managers;
137
138 drmMMListHead named;
139
140 uint64_t gtt_size;
141 int available_fences;
142 int pci_device;
143 unsigned int has_bsd : 1;
144 unsigned int has_blt : 1;
145 unsigned int has_relaxed_fencing : 1;
146 unsigned int has_llc : 1;
147 unsigned int has_wait_timeout : 1;
148 unsigned int bo_reuse : 1;
149 unsigned int no_exec : 1;
150 unsigned int has_vebox : 1;
151 unsigned int has_ext_mmap : 1;
152 bool fenced_relocs;
153
154 struct {
155 void *ptr;
156 uint32_t handle;
157 } userptr_active;
158
159 // manage address for softpin buffer object
160 mos_vma_heap vma_heap[MEMZONE_COUNT];
161 bool use_softpin;
162
163 int device_type;
164 } mos_bufmgr_gem;
165
166 #define DRM_INTEL_RELOC_FENCE (1<<0)
167
168 struct mos_reloc_target {
169 struct mos_linux_bo *bo;
170 int flags;
171 };
172
173 struct mos_softpin_target {
174 struct mos_linux_bo *bo;
175 int flags;
176 };
177
178 struct mos_bo_gem {
179 struct mos_linux_bo bo;
180
181 atomic_t refcount;
182 uint32_t gem_handle;
183 const char *name;
184
185 /**
186 * Kenel-assigned global name for this object
187 *
188 * List contains both flink named and prime fd'd objects
189 */
190 unsigned int global_name;
191 drmMMListHead name_list;
192
193 /**
194 * Index of the buffer within the validation list while preparing a
195 * batchbuffer execution.
196 */
197 int validate_index;
198
199 /**
200 * Current tiling mode
201 */
202 uint32_t tiling_mode;
203 uint32_t swizzle_mode;
204 unsigned long stride;
205
206 time_t free_time;
207
208 /** Array passed to the DRM containing relocation information. */
209 struct drm_i915_gem_relocation_entry *relocs;
210 /**
211 * Array of info structs corresponding to relocs[i].target_handle etc
212 */
213 struct mos_reloc_target *reloc_target_info;
214 /** Number of entries in relocs */
215 int reloc_count;
216 /** Array of BOs that are referenced by this buffer and will be softpinned */
217 struct mos_softpin_target *softpin_target;
218 /** Number softpinned BOs that are referenced by this buffer */
219 int softpin_target_count;
220 /** Maximum amount of softpinned BOs that are referenced by this buffer */
221 int softpin_target_size;
222
223 /** Mapped address for the buffer, saved across map/unmap cycles */
224 void *mem_virtual;
225 /** Uncached Mapped address for the buffer, saved across map/unmap cycles */
226 void *mem_wc_virtual;
227 /** GTT virtual address for the buffer, saved across map/unmap cycles */
228 void *gtt_virtual;
229 /**
230 * Virtual address of the buffer allocated by user, used for userptr
231 * objects only.
232 */
233 void *user_virtual;
234 int map_count;
235
236 /** BO cache list */
237 drmMMListHead head;
238
239 /**
240 * Boolean of whether this BO and its children have been included in
241 * the current drm_intel_bufmgr_check_aperture_space() total.
242 */
243 bool included_in_check_aperture;
244
245 /**
246 * Boolean of whether this buffer has been used as a relocation
247 * target and had its size accounted for, and thus can't have any
248 * further relocations added to it.
249 */
250 bool used_as_reloc_target;
251
252 /**
253 * Boolean of whether we have encountered an error whilst building the relocation tree.
254 */
255 bool has_error;
256
257 /**
258 * Boolean of whether this buffer can be re-used
259 */
260 bool reusable;
261
262 /**
263 * Boolean of whether the GPU is definitely not accessing the buffer.
264 *
265 * This is only valid when reusable, since non-reusable
266 * buffers are those that have been shared wth other
267 * processes, so we don't know their state.
268 */
269 bool idle;
270
271 /**
272 * Boolean of whether this buffer was allocated with userptr
273 */
274 bool is_userptr;
275
276 /**
277 * Boolean of whether this buffer can be placed in the full 48-bit
278 * address range on gen8+.
279 *
280 * By default, buffers will be keep in a 32-bit range, unless this
281 * flag is explicitly set.
282 */
283 bool use_48b_address_range;
284
285 /**
286 * Whether this buffer is softpinned at offset specified by the user
287 */
288 bool is_softpin;
289
290 /*
291 * Whether to remove the dependency of this bo in exebuf.
292 */
293 bool exec_async;
294
295 /**
296 * Size in bytes of this buffer and its relocation descendents.
297 *
298 * Used to avoid costly tree walking in
299 * drm_intel_bufmgr_check_aperture in the common case.
300 */
301 int reloc_tree_size;
302
303 /**
304 * Number of potential fence registers required by this buffer and its
305 * relocations.
306 */
307 int reloc_tree_fences;
308
309 /** Flags that we may need to do the SW_FINSIH ioctl on unmap. */
310 bool mapped_cpu_write;
311
312 /**
313 * Size to pad the object to.
314 *
315 */
316 uint64_t pad_to_size;
317 };
318
319 static unsigned int
320 mos_gem_estimate_batch_space(struct mos_linux_bo ** bo_array, int count);
321
322 static unsigned int
323 mos_gem_compute_batch_space(struct mos_linux_bo ** bo_array, int count);
324
325 static int
326 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
327 uint32_t * swizzle_mode);
328
329 static int
330 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
331 uint32_t tiling_mode,
332 uint32_t stride);
333
334 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
335 time_t time);
336
337 static void mos_gem_bo_unreference(struct mos_linux_bo *bo);
338 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo);
339 static void mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable);
340 static void mos_gem_bo_free(struct mos_linux_bo *bo);
341
to_bo_gem(struct mos_linux_bo * bo)342 static inline struct mos_bo_gem *to_bo_gem(struct mos_linux_bo *bo)
343 {
344 return (struct mos_bo_gem *)bo;
345 }
GetDrmMode()346 static int GetDrmMode()
347 {
348 return 1;//We always use SW Mode in libdrm mock.
349 /*
350 char *penvset_drm = getenv("DRMMode");
351 int DrmMode=0;
352 if(penvset_drm != nullptr)
353 {
354 DrmMode= atoi(penvset_drm);
355 //printf("DrmMode =%d\n", DrmMode);
356 }
357 return DrmMode;
358 */
359 }
360
361 static unsigned long
mos_gem_bo_tile_size(struct mos_bufmgr_gem * bufmgr_gem,unsigned long size,uint32_t * tiling_mode)362 mos_gem_bo_tile_size(struct mos_bufmgr_gem *bufmgr_gem, unsigned long size,
363 uint32_t *tiling_mode)
364 {
365 unsigned long min_size, max_size;
366 unsigned long i;
367
368 if (*tiling_mode == I915_TILING_NONE)
369 return size;
370
371 /* 965+ just need multiples of page size for tiling */
372 return ROUND_UP_TO(size, 4096);
373 }
374
375 /*
376 * Round a given pitch up to the minimum required for X tiling on a
377 * given chip. We use 512 as the minimum to allow for a later tiling
378 * change.
379 */
380 static unsigned long
mos_gem_bo_tile_pitch(struct mos_bufmgr_gem * bufmgr_gem,unsigned long pitch,uint32_t * tiling_mode)381 mos_gem_bo_tile_pitch(struct mos_bufmgr_gem *bufmgr_gem,
382 unsigned long pitch, uint32_t *tiling_mode)
383 {
384 unsigned long tile_width;
385 unsigned long i;
386
387 /* If untiled, then just align it so that we can do rendering
388 * to it with the 3D engine.
389 */
390 if (*tiling_mode == I915_TILING_NONE)
391 return ALIGN(pitch, 64);
392
393 if (*tiling_mode == I915_TILING_X
394 || (IS_915(bufmgr_gem->pci_device)
395 && *tiling_mode == I915_TILING_Y))
396 tile_width = 512;
397 else
398 tile_width = 128;
399
400 /* 965 is flexible */
401 return ROUND_UP_TO(pitch, tile_width);
402 }
403
404 static struct mos_gem_bo_bucket *
mos_gem_bo_bucket_for_size(struct mos_bufmgr_gem * bufmgr_gem,unsigned long size)405 mos_gem_bo_bucket_for_size(struct mos_bufmgr_gem *bufmgr_gem,
406 unsigned long size)
407 {
408 int i;
409
410 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
411 struct mos_gem_bo_bucket *bucket =
412 &bufmgr_gem->cache_bucket[i];
413 if (bucket->size >= size) {
414 return bucket;
415 }
416 }
417
418 return nullptr;
419 }
420
421 static void
mos_gem_dump_validation_list(struct mos_bufmgr_gem * bufmgr_gem)422 mos_gem_dump_validation_list(struct mos_bufmgr_gem *bufmgr_gem)
423 {
424 int i, j;
425
426 for (i = 0; i < bufmgr_gem->exec_count; i++) {
427 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
428 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
429
430 if (bo_gem->relocs == nullptr || bo_gem->softpin_target == nullptr) {
431 MOS_DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,
432 bo_gem->is_softpin ? "*" : "",
433 bo_gem->name);
434 continue;
435 }
436
437 for (j = 0; j < bo_gem->reloc_count; j++) {
438 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[j].bo;
439 struct mos_bo_gem *target_gem =
440 (struct mos_bo_gem *) target_bo;
441
442 MOS_DBG("%2d: %d %s(%s)@0x%08x %08x -> "
443 "%d (%s)@0x%08x %08x + 0x%08x\n",
444 i,
445 bo_gem->gem_handle,
446 bo_gem->is_softpin ? "*" : "",
447 bo_gem->name,
448 upper_32_bits(bo_gem->relocs[j].offset),
449 lower_32_bits(bo_gem->relocs[j].offset),
450 target_gem->gem_handle,
451 target_gem->name,
452 upper_32_bits(target_bo->offset64),
453 lower_32_bits(target_bo->offset64),
454 bo_gem->relocs[j].delta);
455 }
456
457 for (j = 0; j < bo_gem->softpin_target_count; j++) {
458 struct mos_linux_bo *target_bo = bo_gem->softpin_target[j].bo;
459 struct mos_bo_gem *target_gem =
460 (struct mos_bo_gem *) target_bo;
461 MOS_DBG("%2d: %d %s(%s) -> "
462 "%d *(%s)@0x%08x %08x\n",
463 i,
464 bo_gem->gem_handle,
465 bo_gem->is_softpin ? "*" : "",
466 bo_gem->name,
467 target_gem->gem_handle,
468 target_gem->name,
469 upper_32_bits(target_bo->offset64),
470 lower_32_bits(target_bo->offset64));
471 }
472 }
473 }
474
475 static inline void
mos_gem_bo_reference(struct mos_linux_bo * bo)476 mos_gem_bo_reference(struct mos_linux_bo *bo)
477 {
478 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
479
480 atomic_inc(&bo_gem->refcount);
481 }
482
483 /**
484 * Adds the given buffer to the list of buffers to be validated (moved into the
485 * appropriate memory type) with the next batch submission.
486 *
487 * If a buffer is validated multiple times in a batch submission, it ends up
488 * with the intersection of the memory type flags and the union of the
489 * access flags.
490 */
491 static void
mos_add_validate_buffer(struct mos_linux_bo * bo)492 mos_add_validate_buffer(struct mos_linux_bo *bo)
493 {
494 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
495 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
496 int index;
497 struct drm_i915_gem_exec_object *exec_objects;
498 struct mos_linux_bo **exec_bos;
499
500 if (bo_gem->validate_index != -1)
501 return;
502
503 /* Extend the array of validation entries as necessary. */
504 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
505 int new_size = bufmgr_gem->exec_size * 2;
506
507 if (new_size == 0)
508 new_size = 5;
509
510 exec_objects = (struct drm_i915_gem_exec_object *)realloc(bufmgr_gem->exec_objects,
511 sizeof(*bufmgr_gem->exec_objects) * new_size);
512 if (!exec_objects)
513 return;
514
515 bufmgr_gem->exec_objects = exec_objects;
516
517 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
518 sizeof(*bufmgr_gem->exec_bos) * new_size);
519 if (!exec_bos)
520 return;
521
522 bufmgr_gem->exec_bos = exec_bos;
523 bufmgr_gem->exec_size = new_size;
524 }
525
526 index = bufmgr_gem->exec_count;
527 bo_gem->validate_index = index;
528 /* Fill in array entry */
529 bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
530 bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
531 bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;
532 bufmgr_gem->exec_objects[index].alignment = bo->align;
533 bufmgr_gem->exec_objects[index].offset = 0;
534 bufmgr_gem->exec_bos[index] = bo;
535 bufmgr_gem->exec_count++;
536 }
537
538 static void
mos_add_validate_buffer2(struct mos_linux_bo * bo,int need_fence)539 mos_add_validate_buffer2(struct mos_linux_bo *bo, int need_fence)
540 {
541 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
542 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
543 int index;
544 struct drm_i915_gem_exec_object2 *exec2_objects;
545 struct mos_linux_bo **exec_bos;
546 int flags = 0;
547
548 if (need_fence)
549 flags |= EXEC_OBJECT_NEEDS_FENCE;
550 if (bo_gem->pad_to_size)
551 flags |= EXEC_OBJECT_PAD_TO_SIZE;
552 if (bo_gem->use_48b_address_range)
553 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
554 if (bo_gem->is_softpin)
555 flags |= EXEC_OBJECT_PINNED;
556 if (bo_gem->exec_async)
557 flags |= EXEC_OBJECT_ASYNC;
558
559 if (bo_gem->validate_index != -1) {
560 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= flags;
561 return;
562 }
563
564 /* Extend the array of validation entries as necessary. */
565 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
566 int new_size = bufmgr_gem->exec_size * 2;
567
568 if (new_size == 0)
569 new_size = 5;
570 exec2_objects = (struct drm_i915_gem_exec_object2 *)
571 realloc(bufmgr_gem->exec2_objects,
572 sizeof(*bufmgr_gem->exec2_objects) * new_size);
573 if (!exec2_objects)
574 return;
575
576 bufmgr_gem->exec2_objects = exec2_objects;
577
578 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
579 sizeof(*bufmgr_gem->exec_bos) * new_size);
580 if (!exec_bos)
581 return;
582
583 bufmgr_gem->exec_bos = exec_bos;
584 bufmgr_gem->exec_size = new_size;
585 }
586
587 index = bufmgr_gem->exec_count;
588 bo_gem->validate_index = index;
589 /* Fill in array entry */
590 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
591 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
592 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
593 bufmgr_gem->exec2_objects[index].alignment = bo->align;
594 bufmgr_gem->exec2_objects[index].offset = bo_gem->is_softpin ?
595 bo->offset64 : 0;
596 bufmgr_gem->exec_bos[index] = bo;
597 bufmgr_gem->exec2_objects[index].flags = flags;
598 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
599 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
600 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
601 bufmgr_gem->exec_count++;
602 }
603
604 static void
mos_add_reloc_objects(struct mos_reloc_target reloc_target)605 mos_add_reloc_objects(struct mos_reloc_target reloc_target)
606 {
607 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)reloc_target.bo->bufmgr;
608 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)reloc_target.bo;
609 int index;
610 struct drm_i915_gem_exec_object2 *exec2_objects;
611 struct mos_linux_bo **exec_bos;
612
613 if (bo_gem->validate_index != -1) {
614 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= reloc_target.flags;
615 return;
616 }
617
618 /* Extend the array of validation entries as necessary. */
619 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
620 int new_size = bufmgr_gem->exec_size * 2;
621
622 if (new_size == 0)
623 new_size = 5;
624 exec2_objects = (struct drm_i915_gem_exec_object2 *)
625 realloc(bufmgr_gem->exec2_objects,
626 sizeof(*bufmgr_gem->exec2_objects) * new_size);
627 if (!exec2_objects)
628 return;
629
630 bufmgr_gem->exec2_objects = exec2_objects;
631
632 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
633 sizeof(*bufmgr_gem->exec_bos) * new_size);
634 if (!exec_bos)
635 return;
636
637 bufmgr_gem->exec_bos = exec_bos;
638 bufmgr_gem->exec_size = new_size;
639 }
640
641 index = bufmgr_gem->exec_count;
642 bo_gem->validate_index = index;
643 /* Fill in array entry */
644 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
645 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
646 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
647 bufmgr_gem->exec2_objects[index].alignment = reloc_target.bo->align;
648 bufmgr_gem->exec2_objects[index].offset = 0;
649 bufmgr_gem->exec_bos[index] = reloc_target.bo;
650 bufmgr_gem->exec2_objects[index].flags = reloc_target.flags;
651 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
652 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
653 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
654 bufmgr_gem->exec_count++;
655 }
656
657 static void
mos_add_softpin_objects(struct mos_softpin_target softpin_target)658 mos_add_softpin_objects(struct mos_softpin_target softpin_target)
659 {
660 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)softpin_target.bo->bufmgr;
661 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)softpin_target.bo;
662 int index;
663 struct drm_i915_gem_exec_object2 *exec2_objects;
664 struct mos_linux_bo **exec_bos;
665
666 if (bo_gem->validate_index != -1) {
667 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= softpin_target.flags;
668 return;
669 }
670
671 /* Extend the array of validation entries as necessary. */
672 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
673 int new_size = bufmgr_gem->exec_size * 2;
674
675 if (new_size == 0)
676 new_size = 5;
677 exec2_objects = (struct drm_i915_gem_exec_object2 *)
678 realloc(bufmgr_gem->exec2_objects,
679 sizeof(*bufmgr_gem->exec2_objects) * new_size);
680 if (!exec2_objects)
681 return;
682
683 bufmgr_gem->exec2_objects = exec2_objects;
684
685 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
686 sizeof(*bufmgr_gem->exec_bos) * new_size);
687 if (!exec_bos)
688 return;
689
690 bufmgr_gem->exec_bos = exec_bos;
691 bufmgr_gem->exec_size = new_size;
692 }
693
694 index = bufmgr_gem->exec_count;
695 bo_gem->validate_index = index;
696 /* Fill in array entry */
697 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
698 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
699 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
700 bufmgr_gem->exec2_objects[index].alignment = softpin_target.bo->align;
701 bufmgr_gem->exec2_objects[index].offset = softpin_target.bo->offset64;
702 bufmgr_gem->exec2_objects[index].flags = softpin_target.flags;
703 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
704 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
705 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
706 bufmgr_gem->exec_bos[index] = softpin_target.bo;
707 bufmgr_gem->exec_count++;
708 }
709
710 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
711 sizeof(uint32_t))
712
713 static void
mos_bo_gem_set_in_aperture_size(struct mos_bufmgr_gem * bufmgr_gem,struct mos_bo_gem * bo_gem,unsigned int alignment)714 mos_bo_gem_set_in_aperture_size(struct mos_bufmgr_gem *bufmgr_gem,
715 struct mos_bo_gem *bo_gem,
716 unsigned int alignment)
717 {
718 unsigned int size;
719
720 assert(!bo_gem->used_as_reloc_target);
721
722 /* The older chipsets are far-less flexible in terms of tiling,
723 * and require tiled buffer to be size aligned in the aperture.
724 * This means that in the worst possible case we will need a hole
725 * twice as large as the object in order for it to fit into the
726 * aperture. Optimal packing is for wimps.
727 */
728 size = bo_gem->bo.size;
729
730 bo_gem->reloc_tree_size = size + alignment;
731 }
732
733 static int
mos_setup_reloc_list(struct mos_linux_bo * bo)734 mos_setup_reloc_list(struct mos_linux_bo *bo)
735 {
736 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
737 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
738 unsigned int max_relocs = bufmgr_gem->max_relocs;
739
740 if (bo->size / 4 < max_relocs)
741 max_relocs = bo->size / 4;
742
743 bo_gem->relocs = (struct drm_i915_gem_relocation_entry *)malloc(max_relocs *
744 sizeof(struct drm_i915_gem_relocation_entry));
745 bo_gem->reloc_target_info = (struct mos_reloc_target *)malloc(max_relocs *
746 sizeof(struct mos_reloc_target));
747 if (bo_gem->relocs == nullptr || bo_gem->reloc_target_info == nullptr) {
748 bo_gem->has_error = true;
749
750 free (bo_gem->relocs);
751 bo_gem->relocs = nullptr;
752
753 free (bo_gem->reloc_target_info);
754 bo_gem->reloc_target_info = nullptr;
755
756 return 1;
757 }
758
759 return 0;
760 }
761
762 static int
mos_gem_bo_busy(struct mos_linux_bo * bo)763 mos_gem_bo_busy(struct mos_linux_bo *bo)
764 {
765 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
766 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
767 struct drm_i915_gem_busy busy;
768 int ret;
769
770 if (bo_gem->reusable && bo_gem->idle)
771 return false;
772
773 memclear(busy);
774 busy.handle = bo_gem->gem_handle;
775
776 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
777 if (ret == 0) {
778 bo_gem->idle = !busy.busy;
779 return busy.busy;
780 } else {
781 return false;
782 }
783 return (ret == 0 && busy.busy);
784 }
785
786 static int
mos_gem_bo_madvise_internal(struct mos_bufmgr_gem * bufmgr_gem,struct mos_bo_gem * bo_gem,int state)787 mos_gem_bo_madvise_internal(struct mos_bufmgr_gem *bufmgr_gem,
788 struct mos_bo_gem *bo_gem, int state)
789 {
790 struct drm_i915_gem_madvise madv;
791
792 memclear(madv);
793 madv.handle = bo_gem->gem_handle;
794 madv.madv = state;
795 madv.retained = 1;
796 drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
797
798 return madv.retained;
799 }
800
801 static int
mos_gem_bo_madvise(struct mos_linux_bo * bo,int madv)802 mos_gem_bo_madvise(struct mos_linux_bo *bo, int madv)
803 {
804 return mos_gem_bo_madvise_internal
805 ((struct mos_bufmgr_gem *) bo->bufmgr,
806 (struct mos_bo_gem *) bo,
807 madv);
808 }
809
810 /* drop the oldest entries that have been purged by the kernel */
811 static void
mos_gem_bo_cache_purge_bucket(struct mos_bufmgr_gem * bufmgr_gem,struct mos_gem_bo_bucket * bucket)812 mos_gem_bo_cache_purge_bucket(struct mos_bufmgr_gem *bufmgr_gem,
813 struct mos_gem_bo_bucket *bucket)
814 {
815 while (!DRMLISTEMPTY(&bucket->head)) {
816 struct mos_bo_gem *bo_gem;
817
818 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
819 bucket->head.next, head);
820 if (mos_gem_bo_madvise_internal
821 (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))
822 break;
823
824 DRMLISTDEL(&bo_gem->head);
825 mos_gem_bo_free(&bo_gem->bo);
826 }
827 }
828
829 static enum mos_memory_zone
mos_gem_bo_memzone_for_address(uint64_t address)830 mos_gem_bo_memzone_for_address(uint64_t address)
831 {
832 if (address >= MEMZONE_DEVICE_START)
833 return MEMZONE_DEVICE;
834 else
835 return MEMZONE_SYS;
836 }
837 /**
838 * Allocate a section of virtual memory for a buffer, assigning an address.
839 */
840 static uint64_t
mos_gem_bo_vma_alloc(struct mos_bufmgr * bufmgr,enum mos_memory_zone memzone,uint64_t size,uint64_t alignment)841 mos_gem_bo_vma_alloc(struct mos_bufmgr *bufmgr,
842 enum mos_memory_zone memzone,
843 uint64_t size,
844 uint64_t alignment)
845 {
846 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", 0);
847 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
848 /* Force alignment to be some number of pages */
849 alignment = ALIGN(alignment, PAGE_SIZE);
850
851 uint64_t addr = mos_vma_heap_alloc(&bufmgr_gem->vma_heap[memzone], size, alignment);
852
853 // currently only support 48bit range address
854 CHK_CONDITION((addr >> 48ull) != 0, "invalid address, over 48bit range.\n", 0);
855 CHK_CONDITION((addr >> (memzone == MEMZONE_SYS ? 40ull : 41ull)) != 0, "invalid address, over memory zone range.\n", 0);
856 CHK_CONDITION((addr % alignment) != 0, "invalid address, not meet aligment requirement.\n", 0);
857
858 return addr;
859 }
860
861 static void
mos_gem_bo_vma_free(struct mos_bufmgr * bufmgr,uint64_t address,uint64_t size)862 mos_gem_bo_vma_free(struct mos_bufmgr *bufmgr,
863 uint64_t address,
864 uint64_t size)
865 {
866 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", );
867 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
868
869 CHK_CONDITION(address == 0ull, "invalid address.\n", );
870 enum mos_memory_zone memzone = mos_gem_bo_memzone_for_address(address);
871 mos_vma_heap_free(&bufmgr_gem->vma_heap[memzone], address, size);
872 }
873
874 drm_export struct mos_linux_bo *
mos_gem_bo_alloc_internal(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc * alloc)875 mos_gem_bo_alloc_internal(struct mos_bufmgr *bufmgr,
876 struct mos_drm_bo_alloc *alloc)
877 {
878 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
879 struct mos_bo_gem *bo_gem;
880 unsigned int page_size = getpagesize();
881 int ret;
882 struct mos_gem_bo_bucket *bucket;
883 bool alloc_from_cache;
884 unsigned long bo_size;
885 bool for_render = false;
886
887 if (alloc->ext.flags & BO_ALLOC_FOR_RENDER)
888 for_render = true;
889
890 /* Round the allocated size up to a power of two number of pages. */
891 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, alloc->size);
892
893 /* If we don't have caching at this size, don't actually round the
894 * allocation up.
895 */
896 if (bucket == nullptr) {
897 bo_size = alloc->size;
898 if (bo_size < page_size)
899 bo_size = page_size;
900 } else {
901 bo_size = bucket->size;
902 }
903 if(GetDrmMode())//libdrm_mock
904 {
905 pthread_mutex_lock(&bufmgr_gem->lock);
906
907 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
908 if (!bo_gem)
909 return nullptr;
910
911 bo_gem->bo.size = bo_size;
912 bo_gem->bo.handle = -1;
913 bo_gem->bo.bufmgr = bufmgr;
914 bo_gem->bo.align = alloc->alignment;
915 #ifdef __cplusplus
916 bo_gem->bo.virt = malloc(bo_size);
917 bo_gem->mem_virtual = bo_gem->bo.virt;
918 #else
919 bo_gem->bo.virtual = malloc(bo_size);
920 bo_gem->mem_virtual = bo_gem->bo.virtual;
921 #endif
922
923 atomic_set(&bo_gem->refcount, 1);
924 pthread_mutex_unlock(&bufmgr_gem->lock);
925
926 return &bo_gem->bo;
927 }
928
929 pthread_mutex_lock(&bufmgr_gem->lock);
930 /* Get a buffer out of the cache if available */
931 retry:
932 alloc_from_cache = false;
933 if (bucket != nullptr && !DRMLISTEMPTY(&bucket->head)) {
934 if (for_render) {
935 /* Allocate new render-target BOs from the tail (MRU)
936 * of the list, as it will likely be hot in the GPU
937 * cache and in the aperture for us.
938 */
939 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
940 bucket->head.prev, head);
941 DRMLISTDEL(&bo_gem->head);
942 alloc_from_cache = true;
943 bo_gem->bo.align = alloc->alignment;
944 } else {
945 assert(alloc->alignment == 0);
946 /* For non-render-target BOs (where we're probably
947 * going to map it first thing in order to fill it
948 * with data), check if the last BO in the cache is
949 * unbusy, and only reuse in that case. Otherwise,
950 * allocating a new buffer is probably faster than
951 * waiting for the GPU to finish.
952 */
953 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
954 bucket->head.next, head);
955 if (!mos_gem_bo_busy(&bo_gem->bo)) {
956 alloc_from_cache = true;
957 DRMLISTDEL(&bo_gem->head);
958 }
959 }
960
961 if (alloc_from_cache) {
962 if (!mos_gem_bo_madvise_internal
963 (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {
964 mos_gem_bo_free(&bo_gem->bo);
965 mos_gem_bo_cache_purge_bucket(bufmgr_gem,
966 bucket);
967 goto retry;
968 }
969
970 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
971 alloc->ext.tiling_mode,
972 alloc->stride)) {
973 mos_gem_bo_free(&bo_gem->bo);
974 goto retry;
975 }
976 }
977 }
978 pthread_mutex_unlock(&bufmgr_gem->lock);
979
980 if (!alloc_from_cache) {
981 struct drm_i915_gem_create create;
982
983 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
984 if (!bo_gem)
985 return nullptr;
986
987 bo_gem->bo.size = bo_size;
988
989 memclear(create);
990 create.size = bo_size;
991
992 ret = drmIoctl(bufmgr_gem->fd,
993 DRM_IOCTL_I915_GEM_CREATE,
994 &create);
995 bo_gem->gem_handle = create.handle;
996 bo_gem->bo.handle = bo_gem->gem_handle;
997 if (ret != 0) {
998 free(bo_gem);
999 return nullptr;
1000 }
1001 bo_gem->bo.bufmgr = bufmgr;
1002 bo_gem->bo.align = alloc->alignment;
1003
1004 bo_gem->tiling_mode = I915_TILING_NONE;
1005 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1006 bo_gem->stride = 0;
1007
1008 /* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
1009 list (vma_list), so better set the list head here */
1010 DRMINITLISTHEAD(&bo_gem->name_list);
1011 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1012 alloc->ext.tiling_mode,
1013 alloc->stride)) {
1014 mos_gem_bo_free(&bo_gem->bo);
1015 return nullptr;
1016 }
1017 }
1018
1019 bo_gem->name = alloc->name;
1020 atomic_set(&bo_gem->refcount, 1);
1021 bo_gem->validate_index = -1;
1022 bo_gem->reloc_tree_fences = 0;
1023 bo_gem->used_as_reloc_target = false;
1024 bo_gem->has_error = false;
1025 bo_gem->reusable = true;
1026 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1027
1028 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alloc->alignment);
1029
1030 if (bufmgr_gem->use_softpin)
1031 {
1032 mos_bo_set_softpin(&bo_gem->bo);
1033 }
1034
1035 MOS_DBG("bo_create: buf %d (%s) %ldb\n",
1036 bo_gem->gem_handle, bo_gem->name, alloc->size);
1037
1038 return &bo_gem->bo;
1039 }
1040
1041 static struct mos_linux_bo *
mos_gem_bo_alloc(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc * alloc)1042 mos_gem_bo_alloc(struct mos_bufmgr *bufmgr,
1043 struct mos_drm_bo_alloc *alloc)
1044 {
1045 alloc->ext.flags = 0;
1046 alloc->alignment = 0;
1047 alloc->stride = 0;
1048 return mos_gem_bo_alloc_internal(bufmgr, alloc);
1049 }
1050
1051 static struct mos_linux_bo *
mos_gem_bo_alloc_tiled(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_tiled * alloc_tiled)1052 mos_gem_bo_alloc_tiled(struct mos_bufmgr *bufmgr,
1053 struct mos_drm_bo_alloc_tiled *alloc_tiled)
1054 {
1055 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
1056 unsigned long size, stride;
1057 uint32_t tiling;
1058
1059 do {
1060 unsigned long aligned_y, height_alignment;
1061
1062 tiling = alloc_tiled->ext.tiling_mode;
1063
1064 /* If we're tiled, our allocations are in 8 or 32-row blocks,
1065 * so failure to align our height means that we won't allocate
1066 * enough pages.
1067 *
1068 * If we're untiled, we still have to align to 2 rows high
1069 * because the data port accesses 2x2 blocks even if the
1070 * bottom row isn't to be rendered, so failure to align means
1071 * we could walk off the end of the GTT and fault. This is
1072 * documented on 965, and may be the case on older chipsets
1073 * too so we try to be careful.
1074 */
1075 aligned_y = alloc_tiled->y;
1076 height_alignment = 2;
1077
1078 if (tiling == I915_TILING_X
1079 || (IS_915(bufmgr_gem->pci_device)
1080 && tiling == I915_TILING_Y))
1081 height_alignment = 8;
1082 else if (tiling == I915_TILING_Y)
1083 height_alignment = 32;
1084 aligned_y = ALIGN(alloc_tiled->y, height_alignment);
1085
1086 stride = alloc_tiled->x * alloc_tiled->cpp;
1087 stride = mos_gem_bo_tile_pitch(bufmgr_gem, stride, &alloc_tiled->ext.tiling_mode);
1088 size = stride * aligned_y;
1089 size = mos_gem_bo_tile_size(bufmgr_gem, size, &alloc_tiled->ext.tiling_mode);
1090 } while (alloc_tiled->ext.tiling_mode != tiling);
1091 alloc_tiled->pitch = stride;
1092
1093 if (tiling == I915_TILING_NONE)
1094 stride = 0;
1095
1096 struct mos_drm_bo_alloc alloc;
1097 alloc.name = alloc_tiled->name;
1098 alloc.size = size;
1099 alloc.stride = stride;
1100 alloc.ext = alloc_tiled->ext;
1101 return mos_gem_bo_alloc_internal(bufmgr, &alloc);
1102 }
1103
1104 static struct mos_linux_bo *
mos_gem_bo_alloc_userptr(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_userptr * alloc_uptr)1105 mos_gem_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1106 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1107 {
1108 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1109 struct mos_bo_gem *bo_gem;
1110 int ret;
1111 struct drm_i915_gem_userptr userptr;
1112
1113 /* Tiling with userptr surfaces is not supported
1114 * on all hardware so refuse it for time being.
1115 */
1116 if (alloc_uptr->tiling_mode != I915_TILING_NONE)
1117 return nullptr;
1118
1119 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1120 if (!bo_gem)
1121 return nullptr;
1122
1123 bo_gem->bo.size = alloc_uptr->size;
1124
1125 memclear(userptr);
1126 userptr.user_ptr = (__u64)((unsigned long)alloc_uptr->addr);
1127 userptr.user_size = alloc_uptr->size;
1128 userptr.flags = alloc_uptr->flags;
1129
1130 ret = drmIoctl(bufmgr_gem->fd,
1131 DRM_IOCTL_I915_GEM_USERPTR,
1132 &userptr);
1133 if (ret != 0) {
1134 MOS_DBG("bo_create_userptr: "
1135 "ioctl failed with user ptr %p size 0x%lx, "
1136 "user flags 0x%lx\n", alloc_uptr->addr, alloc_uptr->size, alloc_uptr->flags);
1137 free(bo_gem);
1138 return nullptr;
1139 }
1140
1141 bo_gem->gem_handle = userptr.handle;
1142 bo_gem->bo.handle = bo_gem->gem_handle;
1143 bo_gem->bo.bufmgr = bufmgr;
1144 bo_gem->is_userptr = true;
1145 #ifdef __cplusplus
1146 bo_gem->bo.virt = alloc_uptr->addr;
1147 #else
1148 bo_gem->bo.virtual = alloc_uptr->addr;
1149 #endif
1150 /* Save the address provided by user */
1151 bo_gem->user_virtual = alloc_uptr->addr;
1152 bo_gem->tiling_mode = I915_TILING_NONE;
1153 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1154 bo_gem->stride = 0;
1155
1156 DRMINITLISTHEAD(&bo_gem->name_list);
1157
1158 bo_gem->name = alloc_uptr->name;
1159 atomic_set(&bo_gem->refcount, 1);
1160 bo_gem->validate_index = -1;
1161 bo_gem->reloc_tree_fences = 0;
1162 bo_gem->used_as_reloc_target = false;
1163 bo_gem->has_error = false;
1164 bo_gem->reusable = false;
1165 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1166
1167 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1168 if (bufmgr_gem->use_softpin)
1169 {
1170 mos_bo_set_softpin(&bo_gem->bo);
1171 }
1172
1173 MOS_DBG("bo_create_userptr: "
1174 "ptr %p buf %d (%s) size %ldb, stride 0x%x, tile mode %d\n",
1175 alloc_uptr->addr, bo_gem->gem_handle, bo_gem->name,
1176 alloc_uptr->size, alloc_uptr->stride, alloc_uptr->tiling_mode);
1177
1178 return &bo_gem->bo;
1179 }
1180
1181 static bool
has_userptr(struct mos_bufmgr_gem * bufmgr_gem)1182 has_userptr(struct mos_bufmgr_gem *bufmgr_gem)
1183 {
1184 int ret;
1185 void *ptr;
1186 long pgsz;
1187 struct drm_i915_gem_userptr userptr;
1188
1189 pgsz = sysconf(_SC_PAGESIZE);
1190 assert(pgsz > 0);
1191
1192 ret = posix_memalign(&ptr, pgsz, pgsz);
1193 if (ret) {
1194 MOS_DBG("Failed to get a page (%ld) for userptr detection!\n",
1195 pgsz);
1196 return false;
1197 }
1198
1199 memclear(userptr);
1200 userptr.user_ptr = (__u64)(unsigned long)ptr;
1201 userptr.user_size = pgsz;
1202
1203 retry:
1204 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_USERPTR, &userptr);
1205 if (ret) {
1206 if (errno == ENODEV && userptr.flags == 0) {
1207 userptr.flags = I915_USERPTR_UNSYNCHRONIZED;
1208 goto retry;
1209 }
1210 free(ptr);
1211 return false;
1212 }
1213
1214 /* We don't release the userptr bo here as we want to keep the
1215 * kernel mm tracking alive for our lifetime. The first time we
1216 * create a userptr object the kernel has to install a mmu_notifer
1217 * which is a heavyweight operation (e.g. it requires taking all
1218 * mm_locks and stop_machine()).
1219 */
1220
1221 bufmgr_gem->userptr_active.ptr = ptr;
1222 bufmgr_gem->userptr_active.handle = userptr.handle;
1223
1224 return true;
1225 }
1226
1227 static struct mos_linux_bo *
check_bo_alloc_userptr(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_userptr * alloc_uptr)1228 check_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1229 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1230 {
1231 if (has_userptr((struct mos_bufmgr_gem *)bufmgr))
1232 bufmgr->bo_alloc_userptr = mos_gem_bo_alloc_userptr;
1233 else
1234 bufmgr->bo_alloc_userptr = nullptr;
1235
1236 return mos_bo_alloc_userptr(bufmgr, alloc_uptr);
1237 }
1238
1239 /**
1240 * Returns a drm_intel_bo wrapping the given buffer object handle.
1241 *
1242 * This can be used when one application needs to pass a buffer object
1243 * to another.
1244 */
1245 static struct mos_linux_bo *
mos_bufmgr_bo_gem_create_from_name(struct mos_bufmgr * bufmgr,const char * name,unsigned int handle)1246 mos_bufmgr_bo_gem_create_from_name(struct mos_bufmgr *bufmgr,
1247 const char *name,
1248 unsigned int handle)
1249 {
1250 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1251 struct mos_bo_gem *bo_gem;
1252 int ret;
1253 struct drm_gem_open open_arg;
1254 struct drm_i915_gem_get_tiling get_tiling;
1255 drmMMListHead *list;
1256
1257 /* At the moment most applications only have a few named bo.
1258 * For instance, in a DRI client only the render buffers passed
1259 * between X and the client are named. And since X returns the
1260 * alternating names for the front/back buffer a linear search
1261 * provides a sufficiently fast match.
1262 */
1263 pthread_mutex_lock(&bufmgr_gem->lock);
1264 for (list = bufmgr_gem->named.next;
1265 list != &bufmgr_gem->named;
1266 list = list->next) {
1267 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1268 if (bo_gem->global_name == handle) {
1269 mos_gem_bo_reference(&bo_gem->bo);
1270 pthread_mutex_unlock(&bufmgr_gem->lock);
1271 return &bo_gem->bo;
1272 }
1273 }
1274
1275 memclear(open_arg);
1276 open_arg.name = handle;
1277 ret = drmIoctl(bufmgr_gem->fd,
1278 DRM_IOCTL_GEM_OPEN,
1279 &open_arg);
1280 if (ret != 0) {
1281 MOS_DBG("Couldn't reference %s handle 0x%08x: %s\n",
1282 name, handle, strerror(errno));
1283 pthread_mutex_unlock(&bufmgr_gem->lock);
1284 return nullptr;
1285 }
1286 /* Now see if someone has used a prime handle to get this
1287 * object from the kernel before by looking through the list
1288 * again for a matching gem_handle
1289 */
1290 for (list = bufmgr_gem->named.next;
1291 list != &bufmgr_gem->named;
1292 list = list->next) {
1293 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1294 if (bo_gem->gem_handle == open_arg.handle) {
1295 mos_gem_bo_reference(&bo_gem->bo);
1296 pthread_mutex_unlock(&bufmgr_gem->lock);
1297 return &bo_gem->bo;
1298 }
1299 }
1300
1301 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1302 if (!bo_gem) {
1303 pthread_mutex_unlock(&bufmgr_gem->lock);
1304 return nullptr;
1305 }
1306
1307 bo_gem->bo.size = open_arg.size;
1308 bo_gem->bo.offset = 0;
1309 bo_gem->bo.offset64 = 0;
1310 #if defined(__cplusplus)
1311 bo_gem->bo.virt = nullptr;
1312 #else
1313 bo_gem->bo.virtual = nullptr;
1314 #endif
1315 bo_gem->bo.bufmgr = bufmgr;
1316 bo_gem->name = name;
1317 atomic_set(&bo_gem->refcount, 1);
1318 bo_gem->validate_index = -1;
1319 bo_gem->gem_handle = open_arg.handle;
1320 bo_gem->bo.handle = open_arg.handle;
1321 bo_gem->global_name = handle;
1322 bo_gem->reusable = false;
1323 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1324
1325 memclear(get_tiling);
1326 get_tiling.handle = bo_gem->gem_handle;
1327 ret = drmIoctl(bufmgr_gem->fd,
1328 DRM_IOCTL_I915_GEM_GET_TILING,
1329 &get_tiling);
1330 if (ret != 0) {
1331 mos_gem_bo_unreference(&bo_gem->bo);
1332 pthread_mutex_unlock(&bufmgr_gem->lock);
1333 return nullptr;
1334 }
1335 bo_gem->tiling_mode = get_tiling.tiling_mode;
1336 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
1337 /* XXX stride is unknown */
1338 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1339
1340 if (bufmgr_gem->use_softpin)
1341 {
1342 mos_bo_set_softpin(&bo_gem->bo);
1343 }
1344
1345 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
1346 pthread_mutex_unlock(&bufmgr_gem->lock);
1347 MOS_DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
1348
1349 return &bo_gem->bo;
1350 }
1351
1352 static void
mos_gem_bo_free(struct mos_linux_bo * bo)1353 mos_gem_bo_free(struct mos_linux_bo *bo)
1354 {
1355 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1356 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1357 struct drm_gem_close close;
1358 int ret;
1359 if(GetDrmMode())//libdrm_mock
1360 {
1361 free(bo_gem->mem_virtual);
1362 free(bo);
1363 return;
1364 }
1365
1366 if (bo_gem->mem_virtual) {
1367 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, 0));
1368 drm_munmap(bo_gem->mem_virtual, bo_gem->bo.size);
1369 bo_gem->mem_virtual = nullptr;
1370 }
1371 if (bo_gem->gtt_virtual) {
1372 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, 0));
1373 drm_munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
1374 bo_gem->gtt_virtual = nullptr;
1375 }
1376 if (bo_gem->mem_wc_virtual) {
1377 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, 0));
1378 drm_munmap(bo_gem->mem_wc_virtual, bo_gem->bo.size);
1379 bo_gem->mem_wc_virtual = nullptr;
1380 }
1381
1382 /* Close this object */
1383 memclear(close);
1384 close.handle = bo_gem->gem_handle;
1385 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
1386 if (ret != 0) {
1387 MOS_DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
1388 bo_gem->gem_handle, bo_gem->name, strerror(errno));
1389 }
1390
1391 if (bufmgr_gem->use_softpin)
1392 {
1393 /* Return the VMA for reuse */
1394 mos_gem_bo_vma_free(bo->bufmgr, bo->offset64, bo->size);
1395 }
1396 free(bo);
1397 }
1398
1399 static void
mos_gem_bo_mark_mmaps_incoherent(struct mos_linux_bo * bo)1400 mos_gem_bo_mark_mmaps_incoherent(struct mos_linux_bo *bo)
1401 {
1402 #if HAVE_VALGRIND
1403 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1404
1405 if (bo_gem->mem_virtual)
1406 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
1407
1408 if (bo_gem->gtt_virtual)
1409 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, bo->size);
1410
1411 if (bo_gem->mem_wc_virtual)
1412 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, bo->size);
1413 #endif
1414 }
1415
1416 /** Frees all cached buffers significantly older than @time. */
1417 static void
mos_gem_cleanup_bo_cache(struct mos_bufmgr_gem * bufmgr_gem,time_t time)1418 mos_gem_cleanup_bo_cache(struct mos_bufmgr_gem *bufmgr_gem, time_t time)
1419 {
1420 int i;
1421
1422 if (bufmgr_gem->time == time)
1423 return;
1424
1425 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
1426 struct mos_gem_bo_bucket *bucket =
1427 &bufmgr_gem->cache_bucket[i];
1428
1429 while (!DRMLISTEMPTY(&bucket->head)) {
1430 struct mos_bo_gem *bo_gem;
1431
1432 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1433 bucket->head.next, head);
1434 if (time - bo_gem->free_time <= 1)
1435 break;
1436
1437 DRMLISTDEL(&bo_gem->head);
1438
1439 mos_gem_bo_free(&bo_gem->bo);
1440 }
1441 }
1442
1443 bufmgr_gem->time = time;
1444 }
1445
1446 drm_export void
mos_gem_bo_unreference_final(struct mos_linux_bo * bo,time_t time)1447 mos_gem_bo_unreference_final(struct mos_linux_bo *bo, time_t time)
1448 {
1449 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1450 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1451 struct mos_gem_bo_bucket *bucket;
1452 int i;
1453
1454 /* Unreference all the target buffers */
1455 for (i = 0; i < bo_gem->reloc_count; i++) {
1456 if (bo_gem->reloc_target_info[i].bo != bo) {
1457 mos_gem_bo_unreference_locked_timed(bo_gem->
1458 reloc_target_info[i].bo,
1459 time);
1460 }
1461 }
1462 for (i = 0; i < bo_gem->softpin_target_count; i++)
1463 mos_gem_bo_unreference_locked_timed(bo_gem->softpin_target[i].bo,
1464 time);
1465 bo_gem->reloc_count = 0;
1466 bo_gem->used_as_reloc_target = false;
1467 bo_gem->softpin_target_count = 0;
1468 bo_gem->exec_async = false;
1469
1470 MOS_DBG("bo_unreference final: %d (%s)\n",
1471 bo_gem->gem_handle, bo_gem->name);
1472 bo_gem->pad_to_size = 0;
1473
1474 /* release memory associated with this object */
1475 if (bo_gem->reloc_target_info) {
1476 free(bo_gem->reloc_target_info);
1477 bo_gem->reloc_target_info = nullptr;
1478 }
1479 if (bo_gem->relocs) {
1480 free(bo_gem->relocs);
1481 bo_gem->relocs = nullptr;
1482 }
1483 if (bo_gem->softpin_target) {
1484 free(bo_gem->softpin_target);
1485 bo_gem->softpin_target = nullptr;
1486 bo_gem->softpin_target_size = 0;
1487 }
1488 if(GetDrmMode()){
1489 mos_gem_bo_free(bo);
1490 return;
1491 }
1492
1493 /* Clear any left-over mappings */
1494 if (bo_gem->map_count) {
1495 MOS_DBG("bo freed with non-zero map-count %d\n", bo_gem->map_count);
1496 bo_gem->map_count = 0;
1497 mos_gem_bo_mark_mmaps_incoherent(bo);
1498 }
1499
1500 DRMLISTDEL(&bo_gem->name_list);
1501
1502 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
1503 /* Put the buffer into our internal cache for reuse if we can. */
1504 if (bufmgr_gem->bo_reuse && bo_gem->reusable && bucket != nullptr &&
1505 mos_gem_bo_madvise_internal(bufmgr_gem, bo_gem,
1506 I915_MADV_DONTNEED)) {
1507 bo_gem->free_time = time;
1508
1509 bo_gem->name = nullptr;
1510 bo_gem->validate_index = -1;
1511
1512 DRMLISTADDTAIL(&bo_gem->head, &bucket->head);
1513 } else {
1514 mos_gem_bo_free(bo);
1515 }
1516 }
1517
mos_gem_bo_unreference_locked_timed(struct mos_linux_bo * bo,time_t time)1518 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
1519 time_t time)
1520 {
1521 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1522
1523 assert(atomic_read(&bo_gem->refcount) > 0);
1524 if (atomic_dec_and_test(&bo_gem->refcount))
1525 mos_gem_bo_unreference_final(bo, time);
1526 }
1527
mos_gem_bo_unreference(struct mos_linux_bo * bo)1528 static void mos_gem_bo_unreference(struct mos_linux_bo *bo)
1529 {
1530 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1531
1532 assert(atomic_read(&bo_gem->refcount) > 0);
1533
1534 if (atomic_add_unless(&bo_gem->refcount, -1, 1)) {
1535 struct mos_bufmgr_gem *bufmgr_gem =
1536 (struct mos_bufmgr_gem *) bo->bufmgr;
1537 struct timespec time;
1538
1539 clock_gettime(CLOCK_MONOTONIC, &time);
1540
1541 pthread_mutex_lock(&bufmgr_gem->lock);
1542
1543 if (atomic_dec_and_test(&bo_gem->refcount)) {
1544 mos_gem_bo_unreference_final(bo, time.tv_sec);
1545 mos_gem_cleanup_bo_cache(bufmgr_gem, time.tv_sec);
1546 }
1547
1548 pthread_mutex_unlock(&bufmgr_gem->lock);
1549 }
1550 }
1551
1552 static int
map_wc(struct mos_linux_bo * bo)1553 map_wc(struct mos_linux_bo *bo)
1554 {
1555 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1556 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1557 int ret;
1558
1559 if (bo_gem->is_userptr)
1560 return -EINVAL;
1561
1562 if (!bufmgr_gem->has_ext_mmap)
1563 return -EINVAL;
1564
1565 /* Get a mapping of the buffer if we haven't before. */
1566 if (bo_gem->mem_wc_virtual == nullptr) {
1567 struct drm_i915_gem_mmap mmap_arg;
1568
1569 MOS_DBG("bo_map_wc: mmap %d (%s), map_count=%d\n",
1570 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1571
1572 memclear(mmap_arg);
1573 mmap_arg.handle = bo_gem->gem_handle;
1574 /* To indicate the uncached virtual mapping to KMD */
1575 mmap_arg.flags = I915_MMAP_WC;
1576 mmap_arg.size = bo->size;
1577 ret = drmIoctl(bufmgr_gem->fd,
1578 DRM_IOCTL_I915_GEM_MMAP,
1579 &mmap_arg);
1580 if (ret != 0) {
1581 ret = -errno;
1582 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1583 __FILE__, __LINE__, bo_gem->gem_handle,
1584 bo_gem->name, strerror(errno));
1585 return ret;
1586 }
1587 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
1588 bo_gem->mem_wc_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
1589 }
1590 #ifdef __cplusplus
1591 bo->virt = bo_gem->mem_wc_virtual;
1592 #else
1593 bo->virtual = bo_gem->mem_wc_virtual;
1594 #endif
1595
1596 MOS_DBG("bo_map_wc: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
1597 bo_gem->mem_wc_virtual);
1598
1599 return 0;
1600 }
1601
1602 /* To be used in a similar way to mmap_gtt */
1603 drm_export int
mos_gem_bo_map_wc(struct mos_linux_bo * bo)1604 mos_gem_bo_map_wc(struct mos_linux_bo *bo) {
1605 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1606 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1607 struct drm_i915_gem_set_domain set_domain;
1608 int ret;
1609 if(GetDrmMode())//libdrm_mock
1610 {
1611 #ifdef __cplusplus
1612 bo->virt = bo_gem->mem_virtual;
1613 #else
1614 bo->virtual = bo_gem->mem_virtual;
1615 #endif
1616 bo_gem->map_count++;
1617 return 0;
1618 }
1619
1620 pthread_mutex_lock(&bufmgr_gem->lock);
1621
1622 ret = map_wc(bo);
1623 if (ret) {
1624 pthread_mutex_unlock(&bufmgr_gem->lock);
1625 return ret;
1626 }
1627
1628 /* Now move it to the GTT domain so that the GPU and CPU
1629 * caches are flushed and the GPU isn't actively using the
1630 * buffer.
1631 *
1632 * The domain change is done even for the objects which
1633 * are not bounded. For them first the pages are acquired,
1634 * before the domain change.
1635 */
1636 memclear(set_domain);
1637 set_domain.handle = bo_gem->gem_handle;
1638 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
1639 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
1640 ret = drmIoctl(bufmgr_gem->fd,
1641 DRM_IOCTL_I915_GEM_SET_DOMAIN,
1642 &set_domain);
1643 if (ret != 0) {
1644 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
1645 __FILE__, __LINE__, bo_gem->gem_handle,
1646 strerror(errno));
1647 }
1648 mos_gem_bo_mark_mmaps_incoherent(bo);
1649 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_wc_virtual, bo->size));
1650 pthread_mutex_unlock(&bufmgr_gem->lock);
1651
1652 return 0;
1653 }
1654
mos_gem_bo_map(struct mos_linux_bo * bo,int write_enable)1655 drm_export int mos_gem_bo_map(struct mos_linux_bo *bo, int write_enable)
1656 {
1657 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1658 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1659 struct drm_i915_gem_set_domain set_domain;
1660 int ret;
1661 if(GetDrmMode())//libdrm_mock
1662 {
1663 #ifdef __cplusplus
1664 bo->virt = bo_gem->mem_virtual;
1665 #else
1666 bo->virtual = bo_gem->mem_virtual;
1667 #endif
1668 bo_gem->map_count++;
1669 return 0;
1670 }
1671
1672 if (bo_gem->is_userptr) {
1673 /* Return the same user ptr */
1674 #ifdef __cplusplus
1675 bo->virt = bo_gem->user_virtual;
1676 #else
1677 bo->virtual = bo_gem->user_virtual;
1678 #endif
1679 return 0;
1680 }
1681
1682 pthread_mutex_lock(&bufmgr_gem->lock);
1683
1684 if (!bo_gem->mem_virtual) {
1685 struct drm_i915_gem_mmap mmap_arg;
1686
1687 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
1688 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1689
1690 memclear(mmap_arg);
1691 mmap_arg.handle = bo_gem->gem_handle;
1692 mmap_arg.size = bo->size;
1693 ret = drmIoctl(bufmgr_gem->fd,
1694 DRM_IOCTL_I915_GEM_MMAP,
1695 &mmap_arg);
1696 if (ret != 0) {
1697 ret = -errno;
1698 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1699 __FILE__, __LINE__, bo_gem->gem_handle,
1700 bo_gem->name, strerror(errno));
1701 pthread_mutex_unlock(&bufmgr_gem->lock);
1702 return ret;
1703 }
1704 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
1705 bo_gem->mem_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
1706 }
1707 MOS_DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
1708 bo_gem->mem_virtual);
1709 #ifdef __cplusplus
1710 bo->virt = bo_gem->mem_virtual;
1711 #else
1712 bo->virtual = bo_gem->mem_virtual;
1713 #endif
1714
1715 memclear(set_domain);
1716 set_domain.handle = bo_gem->gem_handle;
1717 set_domain.read_domains = I915_GEM_DOMAIN_CPU;
1718 if (write_enable)
1719 set_domain.write_domain = I915_GEM_DOMAIN_CPU;
1720 else
1721 set_domain.write_domain = 0;
1722 ret = drmIoctl(bufmgr_gem->fd,
1723 DRM_IOCTL_I915_GEM_SET_DOMAIN,
1724 &set_domain);
1725 if (ret != 0) {
1726 MOS_DBG("%s:%d: Error setting to CPU domain %d: %s\n",
1727 __FILE__, __LINE__, bo_gem->gem_handle,
1728 strerror(errno));
1729 }
1730
1731 if (write_enable)
1732 bo_gem->mapped_cpu_write = true;
1733
1734 mos_gem_bo_mark_mmaps_incoherent(bo);
1735 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
1736 pthread_mutex_unlock(&bufmgr_gem->lock);
1737
1738 return 0;
1739 }
1740
1741 drm_export int
map_gtt(struct mos_linux_bo * bo)1742 map_gtt(struct mos_linux_bo *bo)
1743 {
1744 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1745 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1746 int ret;
1747 if(GetDrmMode())//libdrm_mock
1748 {
1749 #ifdef __cplusplus
1750 bo->virt = bo_gem->mem_virtual;
1751 #else
1752 bo->virtual = bo_gem->mem_virtual;
1753 #endif
1754 bo_gem->map_count++;
1755 return 0;
1756 }
1757
1758 if (bo_gem->is_userptr)
1759 return -EINVAL;
1760
1761 /* Get a mapping of the buffer if we haven't before. */
1762 if (bo_gem->gtt_virtual == nullptr) {
1763 struct drm_i915_gem_mmap_gtt mmap_arg;
1764
1765 MOS_DBG("bo_map_gtt: mmap %d (%s), map_count=%d\n",
1766 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1767
1768 memclear(mmap_arg);
1769 mmap_arg.handle = bo_gem->gem_handle;
1770
1771 /* Get the fake offset back... */
1772 ret = drmIoctl(bufmgr_gem->fd,
1773 DRM_IOCTL_I915_GEM_MMAP_GTT,
1774 &mmap_arg);
1775 if (ret != 0) {
1776 ret = -errno;
1777 MOS_DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
1778 __FILE__, __LINE__,
1779 bo_gem->gem_handle, bo_gem->name,
1780 strerror(errno));
1781 return ret;
1782 }
1783
1784 /* and mmap it */
1785 bo_gem->gtt_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1786 MAP_SHARED, bufmgr_gem->fd,
1787 mmap_arg.offset);
1788 if (bo_gem->gtt_virtual == MAP_FAILED) {
1789 bo_gem->gtt_virtual = nullptr;
1790 ret = -errno;
1791 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1792 __FILE__, __LINE__,
1793 bo_gem->gem_handle, bo_gem->name,
1794 strerror(errno));
1795 return ret;
1796 }
1797 }
1798 #ifdef __cplusplus
1799 bo->virt = bo_gem->gtt_virtual;
1800 #else
1801 bo->virtual = bo_gem->gtt_virtual;
1802 #endif
1803
1804 MOS_DBG("bo_map_gtt: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
1805 bo_gem->gtt_virtual);
1806
1807 return 0;
1808 }
1809
1810 static int
mos_gem_bo_map_gtt(struct mos_linux_bo * bo)1811 mos_gem_bo_map_gtt(struct mos_linux_bo *bo)
1812 {
1813 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1814 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1815 struct drm_i915_gem_set_domain set_domain;
1816 int ret;
1817
1818 pthread_mutex_lock(&bufmgr_gem->lock);
1819
1820 ret = map_gtt(bo);
1821 if (ret) {
1822 pthread_mutex_unlock(&bufmgr_gem->lock);
1823 return ret;
1824 }
1825
1826 /* Now move it to the GTT domain so that the GPU and CPU
1827 * caches are flushed and the GPU isn't actively using the
1828 * buffer.
1829 *
1830 * The pagefault handler does this domain change for us when
1831 * it has unbound the BO from the GTT, but it's up to us to
1832 * tell it when we're about to use things if we had done
1833 * rendering and it still happens to be bound to the GTT.
1834 */
1835 memclear(set_domain);
1836 set_domain.handle = bo_gem->gem_handle;
1837 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
1838 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
1839 ret = drmIoctl(bufmgr_gem->fd,
1840 DRM_IOCTL_I915_GEM_SET_DOMAIN,
1841 &set_domain);
1842 if (ret != 0) {
1843 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
1844 __FILE__, __LINE__, bo_gem->gem_handle,
1845 strerror(errno));
1846 }
1847
1848 mos_gem_bo_mark_mmaps_incoherent(bo);
1849 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
1850 pthread_mutex_unlock(&bufmgr_gem->lock);
1851
1852 return 0;
1853 }
1854
1855 /**
1856 * Performs a mapping of the buffer object like the normal GTT
1857 * mapping, but avoids waiting for the GPU to be done reading from or
1858 * rendering to the buffer.
1859 *
1860 * This is used in the implementation of GL_ARB_map_buffer_range: The
1861 * user asks to create a buffer, then does a mapping, fills some
1862 * space, runs a drawing command, then asks to map it again without
1863 * synchronizing because it guarantees that it won't write over the
1864 * data that the GPU is busy using (or, more specifically, that if it
1865 * does write over the data, it acknowledges that rendering is
1866 * undefined).
1867 */
1868
1869 static int
mos_gem_bo_map_unsynchronized(struct mos_linux_bo * bo)1870 mos_gem_bo_map_unsynchronized(struct mos_linux_bo *bo)
1871 {
1872 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1873 #ifdef HAVE_VALGRIND
1874 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1875 #endif
1876 int ret;
1877
1878 /* If the CPU cache isn't coherent with the GTT, then use a
1879 * regular synchronized mapping. The problem is that we don't
1880 * track where the buffer was last used on the CPU side in
1881 * terms of drm_intel_bo_map vs drm_intel_gem_bo_map_gtt, so
1882 * we would potentially corrupt the buffer even when the user
1883 * does reasonable things.
1884 */
1885 if (!bufmgr_gem->has_llc)
1886 return mos_gem_bo_map_gtt(bo);
1887
1888 pthread_mutex_lock(&bufmgr_gem->lock);
1889
1890 ret = map_gtt(bo);
1891 if (ret == 0) {
1892 mos_gem_bo_mark_mmaps_incoherent(bo);
1893 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
1894 }
1895
1896 pthread_mutex_unlock(&bufmgr_gem->lock);
1897
1898 return ret;
1899 }
1900
mos_gem_bo_unmap(struct mos_linux_bo * bo)1901 drm_export int mos_gem_bo_unmap(struct mos_linux_bo *bo)
1902 {
1903 struct mos_bufmgr_gem *bufmgr_gem;
1904 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1905 int ret = 0;
1906
1907 if (bo == nullptr)
1908 return 0;
1909
1910 if (bo_gem->is_userptr)
1911 return 0;
1912
1913 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1914 if(GetDrmMode())//libdrm_mock
1915 {
1916 --bo_gem->map_count ;
1917 return 0;
1918 }
1919
1920 pthread_mutex_lock(&bufmgr_gem->lock);
1921
1922 if (bo_gem->map_count <= 0) {
1923 MOS_DBG("attempted to unmap an unmapped bo\n");
1924 pthread_mutex_unlock(&bufmgr_gem->lock);
1925 /* Preserve the old behaviour of just treating this as a
1926 * no-op rather than reporting the error.
1927 */
1928 return 0;
1929 }
1930
1931 if (bo_gem->mapped_cpu_write) {
1932 struct drm_i915_gem_sw_finish sw_finish;
1933
1934 /* Cause a flush to happen if the buffer's pinned for
1935 * scanout, so the results show up in a timely manner.
1936 * Unlike GTT set domains, this only does work if the
1937 * buffer should be scanout-related.
1938 */
1939 memclear(sw_finish);
1940 sw_finish.handle = bo_gem->gem_handle;
1941 ret = drmIoctl(bufmgr_gem->fd,
1942 DRM_IOCTL_I915_GEM_SW_FINISH,
1943 &sw_finish);
1944 ret = ret == -1 ? -errno : 0;
1945
1946 bo_gem->mapped_cpu_write = false;
1947 }
1948
1949 /* We need to unmap after every innovation as we cannot track
1950 * an open vma for every bo as that will exhaasut the system
1951 * limits and cause later failures.
1952 */
1953 if (--bo_gem->map_count == 0) {
1954 mos_gem_bo_mark_mmaps_incoherent(bo);
1955 #ifdef __cplusplus
1956 bo->virt = nullptr;
1957 #else
1958 bo->virtual = nullptr;
1959 #endif
1960 }
1961 pthread_mutex_unlock(&bufmgr_gem->lock);
1962
1963 return ret;
1964 }
1965
1966 static int
mos_gem_bo_unmap_wc(struct mos_linux_bo * bo)1967 mos_gem_bo_unmap_wc(struct mos_linux_bo *bo)
1968 {
1969 return mos_gem_bo_unmap(bo);
1970 }
1971
1972 static int
mos_gem_bo_unmap_gtt(struct mos_linux_bo * bo)1973 mos_gem_bo_unmap_gtt(struct mos_linux_bo *bo)
1974 {
1975 return mos_gem_bo_unmap(bo);
1976 }
1977
1978 /** Waits for all GPU rendering with the object to have completed. */
1979 static void
mos_gem_bo_wait_rendering(struct mos_linux_bo * bo)1980 mos_gem_bo_wait_rendering(struct mos_linux_bo *bo)
1981 {
1982 mos_gem_bo_start_gtt_access(bo, 1);
1983 }
1984
1985 /**
1986 * Waits on a BO for the given amount of time.
1987 *
1988 * @bo: buffer object to wait for
1989 * @timeout_ns: amount of time to wait in nanoseconds.
1990 * If value is less than 0, an infinite wait will occur.
1991 *
1992 * Returns 0 if the wait was successful ie. the last batch referencing the
1993 * object has completed within the allotted time. Otherwise some negative return
1994 * value describes the error. Of particular interest is -ETIME when the wait has
1995 * failed to yield the desired result.
1996 *
1997 * Similar to drm_intel_gem_bo_wait_rendering except a timeout parameter allows
1998 * the operation to give up after a certain amount of time. Another subtle
1999 * difference is the internal locking semantics are different (this variant does
2000 * not hold the lock for the duration of the wait). This makes the wait subject
2001 * to a larger userspace race window.
2002 *
2003 * The implementation shall wait until the object is no longer actively
2004 * referenced within a batch buffer at the time of the call. The wait will
2005 * not guarantee that the buffer is re-issued via another thread, or an flinked
2006 * handle. Userspace must make sure this race does not occur if such precision
2007 * is important.
2008 *
2009 * Note that some kernels have broken the inifite wait for negative values
2010 * promise, upgrade to latest stable kernels if this is the case.
2011 */
2012 static int
mos_gem_bo_wait(struct mos_linux_bo * bo,int64_t timeout_ns)2013 mos_gem_bo_wait(struct mos_linux_bo *bo, int64_t timeout_ns)
2014 {
2015 if(GetDrmMode())
2016 return 0; //libdrm_mock
2017
2018 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2019 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2020 struct drm_i915_gem_wait wait;
2021 int ret;
2022
2023 if (!bufmgr_gem->has_wait_timeout) {
2024 MOS_DBG("%s:%d: Timed wait is not supported. Falling back to "
2025 "infinite wait\n", __FILE__, __LINE__);
2026 if (timeout_ns) {
2027 mos_gem_bo_wait_rendering(bo);
2028 return 0;
2029 } else {
2030 return mos_gem_bo_busy(bo) ? -ETIME : 0;
2031 }
2032 }
2033
2034 memclear(wait);
2035 wait.bo_handle = bo_gem->gem_handle;
2036 wait.timeout_ns = timeout_ns;
2037 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2038 if (ret == -1)
2039 return -errno;
2040
2041 return ret;
2042 }
2043
2044 /**
2045 * Sets the object to the GTT read and possibly write domain, used by the X
2046 * 2D driver in the absence of kernel support to do drm_intel_gem_bo_map_gtt().
2047 *
2048 * In combination with drm_intel_gem_bo_pin() and manual fence management, we
2049 * can do tiled pixmaps this way.
2050 */
2051 static void
mos_gem_bo_start_gtt_access(struct mos_linux_bo * bo,int write_enable)2052 mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable)
2053 {
2054 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2055 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2056 struct drm_i915_gem_set_domain set_domain;
2057 int ret;
2058
2059 memclear(set_domain);
2060 set_domain.handle = bo_gem->gem_handle;
2061 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2062 set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_GTT : 0;
2063 ret = drmIoctl(bufmgr_gem->fd,
2064 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2065 &set_domain);
2066 if (ret != 0) {
2067 MOS_DBG("%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
2068 __FILE__, __LINE__, bo_gem->gem_handle,
2069 set_domain.read_domains, set_domain.write_domain,
2070 strerror(errno));
2071 }
2072 }
2073
2074 static void
mos_bufmgr_gem_destroy(struct mos_bufmgr * bufmgr)2075 mos_bufmgr_gem_destroy(struct mos_bufmgr *bufmgr)
2076 {
2077 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
2078 struct drm_gem_close close_bo;
2079 int i, ret;
2080
2081 free(bufmgr_gem->exec2_objects);
2082 free(bufmgr_gem->exec_objects);
2083 free(bufmgr_gem->exec_bos);
2084
2085 pthread_mutex_destroy(&bufmgr_gem->lock);
2086
2087 /* Free any cached buffer objects we were going to reuse */
2088 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
2089 struct mos_gem_bo_bucket *bucket =
2090 &bufmgr_gem->cache_bucket[i];
2091 struct mos_bo_gem *bo_gem;
2092
2093 while (!DRMLISTEMPTY(&bucket->head)) {
2094 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
2095 bucket->head.next, head);
2096 DRMLISTDEL(&bo_gem->head);
2097
2098 mos_gem_bo_free(&bo_gem->bo);
2099 }
2100 }
2101
2102 /* Release userptr bo kept hanging around for optimisation. */
2103 if (bufmgr_gem->userptr_active.ptr) {
2104 memclear(close_bo);
2105 close_bo.handle = bufmgr_gem->userptr_active.handle;
2106 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close_bo);
2107 free(bufmgr_gem->userptr_active.ptr);
2108 if (ret)
2109 fprintf(stderr,
2110 "Failed to release test userptr object! (%d) "
2111 "i915 kernel driver may not be sane!\n", errno);
2112 }
2113
2114 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_SYS]);
2115 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_DEVICE]);
2116 free(bufmgr);
2117 }
2118
2119 static int
do_bo_emit_reloc(struct mos_linux_bo * bo,uint32_t offset,struct mos_linux_bo * target_bo,uint32_t target_offset,uint32_t read_domains,uint32_t write_domain,bool need_fence,uint64_t presumed_offset)2120 do_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2121 struct mos_linux_bo *target_bo, uint32_t target_offset,
2122 uint32_t read_domains, uint32_t write_domain,
2123 bool need_fence, uint64_t presumed_offset)
2124 {
2125 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2126 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2127 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2128
2129 if (bo_gem->has_error)
2130 return -ENOMEM;
2131
2132 if (target_bo_gem->has_error) {
2133 bo_gem->has_error = true;
2134 return -ENOMEM;
2135 }
2136
2137 /* Create a new relocation list if needed */
2138 if (bo_gem->relocs == nullptr && mos_setup_reloc_list(bo))
2139 return -ENOMEM;
2140
2141 /* Check overflow */
2142 assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
2143
2144 /* Check args */
2145 assert(offset <= bo->size - 4);
2146 assert((write_domain & (write_domain - 1)) == 0);
2147
2148 /* An object needing a fence is a tiled buffer, so it won't have
2149 * relocs to other buffers.
2150 */
2151 if (need_fence) {
2152 assert(target_bo_gem->reloc_count == 0);
2153 target_bo_gem->reloc_tree_fences = 1;
2154 }
2155
2156 /* Make sure that we're not adding a reloc to something whose size has
2157 * already been accounted for.
2158 */
2159 assert(!bo_gem->used_as_reloc_target);
2160 if (target_bo_gem != bo_gem) {
2161 target_bo_gem->used_as_reloc_target = true;
2162 bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
2163 bo_gem->reloc_tree_fences += target_bo_gem->reloc_tree_fences;
2164 }
2165
2166 int flags = 0;
2167 if (target_bo_gem->pad_to_size)
2168 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2169 if (target_bo_gem->use_48b_address_range)
2170 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2171 if (target_bo_gem->exec_async)
2172 flags |= EXEC_OBJECT_ASYNC;
2173
2174 if (target_bo != bo)
2175 mos_gem_bo_reference(target_bo);
2176
2177 bo_gem->reloc_target_info[bo_gem->reloc_count].bo = target_bo;
2178 bo_gem->reloc_target_info[bo_gem->reloc_count].flags = flags;
2179 bo_gem->relocs[bo_gem->reloc_count].offset = offset;
2180 bo_gem->relocs[bo_gem->reloc_count].delta = target_offset;
2181 bo_gem->relocs[bo_gem->reloc_count].target_handle =
2182 target_bo_gem->gem_handle;
2183 bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
2184 bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
2185 bo_gem->relocs[bo_gem->reloc_count].presumed_offset = presumed_offset;
2186 bo_gem->reloc_count++;
2187
2188 return 0;
2189 }
2190
2191 static void
mos_gem_bo_use_48b_address_range(struct mos_linux_bo * bo,uint32_t enable)2192 mos_gem_bo_use_48b_address_range(struct mos_linux_bo *bo, uint32_t enable)
2193 {
2194 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2195 bo_gem->use_48b_address_range = enable;
2196 }
2197
2198 static void
mos_gem_bo_set_object_async(struct mos_linux_bo * bo)2199 mos_gem_bo_set_object_async(struct mos_linux_bo *bo)
2200 {
2201 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2202 bo_gem->exec_async = true;
2203 }
2204
2205 static void
mos_gem_bo_set_exec_object_async(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo)2206 mos_gem_bo_set_exec_object_async(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
2207 {
2208 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2209 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2210 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2211 int i;
2212 for (i = 0; i < bo_gem->softpin_target_count; i++)
2213 {
2214 if (bo_gem->softpin_target[i].bo == target_bo)
2215 {
2216 bo_gem->softpin_target[i].flags |= EXEC_OBJECT_ASYNC;
2217 break;
2218 }
2219 }
2220 }
2221
2222 static int
mos_gem_bo_add_softpin_target(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo,bool write_flag)2223 mos_gem_bo_add_softpin_target(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo, bool write_flag)
2224 {
2225 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2226 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2227 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2228 if (bo_gem->has_error)
2229 return -ENOMEM;
2230
2231 if (target_bo_gem->has_error) {
2232 bo_gem->has_error = true;
2233 return -ENOMEM;
2234 }
2235
2236 if (!target_bo_gem->is_softpin)
2237 return -EINVAL;
2238 if (target_bo_gem == bo_gem)
2239 return -EINVAL;
2240
2241 if (bo_gem->softpin_target_count == bo_gem->softpin_target_size) {
2242 int new_size = bo_gem->softpin_target_size * 2;
2243 if (new_size == 0)
2244 new_size = bufmgr_gem->max_relocs;
2245
2246 bo_gem->softpin_target = (struct mos_softpin_target *)realloc(bo_gem->softpin_target, new_size *
2247 sizeof(struct mos_softpin_target));
2248 if (!bo_gem->softpin_target)
2249 return -ENOMEM;
2250
2251 bo_gem->softpin_target_size = new_size;
2252 }
2253
2254 int flags = EXEC_OBJECT_PINNED;
2255 if (target_bo_gem->pad_to_size)
2256 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2257 if (target_bo_gem->use_48b_address_range)
2258 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2259 if (target_bo_gem->exec_async)
2260 flags |= EXEC_OBJECT_ASYNC;
2261 if (write_flag)
2262 flags |= EXEC_OBJECT_WRITE;
2263
2264 bo_gem->softpin_target[bo_gem->softpin_target_count].bo = target_bo;
2265 bo_gem->softpin_target[bo_gem->softpin_target_count].flags = flags;
2266 mos_gem_bo_reference(target_bo);
2267 bo_gem->softpin_target_count++;
2268
2269 return 0;
2270 }
2271
2272 static int
mos_gem_bo_pad_to_size(struct mos_linux_bo * bo,uint64_t pad_to_size)2273 mos_gem_bo_pad_to_size(struct mos_linux_bo *bo, uint64_t pad_to_size)
2274 {
2275 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2276
2277 if (pad_to_size && pad_to_size < bo->size)
2278 return -EINVAL;
2279
2280 bo_gem->pad_to_size = pad_to_size;
2281 return 0;
2282 }
2283
2284 static int
mos_gem_bo_emit_reloc(struct mos_linux_bo * bo,uint32_t offset,struct mos_linux_bo * target_bo,uint32_t target_offset,uint32_t read_domains,uint32_t write_domain,uint64_t presumed_offset)2285 mos_gem_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2286 struct mos_linux_bo *target_bo, uint32_t target_offset,
2287 uint32_t read_domains, uint32_t write_domain,
2288 uint64_t presumed_offset)
2289 {
2290 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2291
2292 return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
2293 read_domains, write_domain,
2294 false,
2295 presumed_offset);
2296 }
2297
2298 /**
2299 * Removes existing relocation entries in the BO after "start".
2300 *
2301 * This allows a user to avoid a two-step process for state setup with
2302 * counting up all the buffer objects and doing a
2303 * drm_intel_bufmgr_check_aperture_space() before emitting any of the
2304 * relocations for the state setup. Instead, save the state of the
2305 * batchbuffer including drm_intel_gem_get_reloc_count(), emit all the
2306 * state, and then check if it still fits in the aperture.
2307 *
2308 * Any further drm_intel_bufmgr_check_aperture_space() queries
2309 * involving this buffer in the tree are undefined after this call.
2310 *
2311 * This also removes all softpinned targets being referenced by the BO.
2312 */
2313 static void
mos_gem_bo_clear_relocs(struct mos_linux_bo * bo,int start)2314 mos_gem_bo_clear_relocs(struct mos_linux_bo *bo, int start)
2315 {
2316 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2317 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2318 int i;
2319 struct timespec time;
2320 if(GetDrmMode())
2321 return; //libdrm_mock
2322
2323 clock_gettime(CLOCK_MONOTONIC, &time);
2324
2325 assert(bo_gem->reloc_count >= start);
2326
2327 /* Unreference the cleared target buffers */
2328 pthread_mutex_lock(&bufmgr_gem->lock);
2329
2330 for (i = start; i < bo_gem->reloc_count; i++) {
2331 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->reloc_target_info[i].bo;
2332 if (&target_bo_gem->bo != bo) {
2333 bo_gem->reloc_tree_fences -= target_bo_gem->reloc_tree_fences;
2334 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo,
2335 time.tv_sec);
2336 }
2337 }
2338 bo_gem->reloc_count = start;
2339
2340 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2341 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->softpin_target[i].bo;
2342 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo, time.tv_sec);
2343 }
2344 bo_gem->softpin_target_count = 0;
2345
2346 pthread_mutex_unlock(&bufmgr_gem->lock);
2347
2348 }
2349
2350 /**
2351 * Walk the tree of relocations rooted at BO and accumulate the list of
2352 * validations to be performed and update the relocation buffers with
2353 * index values into the validation list.
2354 */
2355 static void
mos_gem_bo_process_reloc(struct mos_linux_bo * bo)2356 mos_gem_bo_process_reloc(struct mos_linux_bo *bo)
2357 {
2358 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2359 int i;
2360
2361 if (bo_gem->relocs == nullptr)
2362 return;
2363
2364 for (i = 0; i < bo_gem->reloc_count; i++) {
2365 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2366
2367 if (target_bo == bo)
2368 continue;
2369
2370 mos_gem_bo_mark_mmaps_incoherent(bo);
2371
2372 /* Continue walking the tree depth-first. */
2373 mos_gem_bo_process_reloc(target_bo);
2374
2375 /* Add the target to the validate list */
2376 mos_add_validate_buffer(target_bo);
2377 }
2378 }
2379
2380 static void
mos_gem_bo_process_reloc2(struct mos_linux_bo * bo)2381 mos_gem_bo_process_reloc2(struct mos_linux_bo *bo)
2382 {
2383 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2384 int i;
2385
2386 if (bo_gem->relocs == nullptr && bo_gem->softpin_target == nullptr)
2387 return;
2388
2389 for (i = 0; i < bo_gem->reloc_count; i++) {
2390 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2391 int need_fence;
2392
2393 if (target_bo == bo)
2394 continue;
2395
2396 mos_gem_bo_mark_mmaps_incoherent(bo);
2397
2398 /* Continue walking the tree depth-first. */
2399 mos_gem_bo_process_reloc2(target_bo);
2400
2401 need_fence = (bo_gem->reloc_target_info[i].flags &
2402 DRM_INTEL_RELOC_FENCE);
2403
2404 /* Add the target to the validate list */
2405 mos_add_validate_buffer2(target_bo, need_fence);
2406 }
2407
2408 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2409 struct mos_linux_bo *target_bo = bo_gem->softpin_target[i].bo;
2410
2411 if (target_bo == bo)
2412 continue;
2413
2414 mos_gem_bo_mark_mmaps_incoherent(bo);
2415 mos_gem_bo_process_reloc2(target_bo);
2416 mos_add_validate_buffer2(target_bo, false);
2417 }
2418 }
2419
2420 static void
mos_update_buffer_offsets(struct mos_bufmgr_gem * bufmgr_gem)2421 mos_update_buffer_offsets(struct mos_bufmgr_gem *bufmgr_gem)
2422 {
2423 int i;
2424
2425 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2426 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2427 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2428
2429 /* Update the buffer offset */
2430 if (bufmgr_gem->exec_objects[i].offset != bo->offset64) {
2431 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2432 bo_gem->gem_handle, bo_gem->name,
2433 upper_32_bits(bo->offset64),
2434 lower_32_bits(bo->offset64),
2435 upper_32_bits(bufmgr_gem->exec_objects[i].offset),
2436 lower_32_bits(bufmgr_gem->exec_objects[i].offset));
2437 bo->offset64 = bufmgr_gem->exec_objects[i].offset;
2438 bo->offset = bufmgr_gem->exec_objects[i].offset;
2439 }
2440 }
2441 }
2442
2443 static void
mos_update_buffer_offsets2(struct mos_bufmgr_gem * bufmgr_gem,mos_linux_context * ctx,mos_linux_bo * cmd_bo)2444 mos_update_buffer_offsets2 (struct mos_bufmgr_gem *bufmgr_gem, mos_linux_context *ctx, mos_linux_bo *cmd_bo)
2445 {
2446 int i;
2447
2448 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2449 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2450 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2451
2452 /* Update the buffer offset */
2453 if (bufmgr_gem->exec2_objects[i].offset != bo->offset64) {
2454 /* If we're seeing softpinned object here it means that the kernel
2455 * has relocated our object... Indicating a programming error
2456 */
2457 assert(!bo_gem->is_softpin);
2458 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2459 bo_gem->gem_handle, bo_gem->name,
2460 upper_32_bits(bo->offset64),
2461 lower_32_bits(bo->offset64),
2462 upper_32_bits(bufmgr_gem->exec2_objects[i].offset),
2463 lower_32_bits(bufmgr_gem->exec2_objects[i].offset));
2464 bo->offset64 = bufmgr_gem->exec2_objects[i].offset;
2465 bo->offset = bufmgr_gem->exec2_objects[i].offset;
2466 }
2467
2468 #if 0//ndef ANDROID
2469 if (cmd_bo != bo) {
2470 auto item_ctx = ctx->pOsContext->contextOffsetList.begin();
2471 for (; item_ctx != ctx->pOsContext->contextOffsetList.end(); item_ctx++) {
2472 if (item_ctx->intel_context == ctx && item_ctx->target_bo == bo) {
2473 item_ctx->offset64 = bo->offset64;
2474 break;
2475 }
2476 }
2477 if ( item_ctx == ctx->pOsContext->contextOffsetList.end()) {
2478 struct MOS_CONTEXT_OFFSET newContext = {ctx,
2479 bo,
2480 bo->offset64};
2481 ctx->pOsContext->contextOffsetList.push_back(newContext);
2482 }
2483 }
2484 #endif
2485 }
2486 }
2487
2488 drm_export int
do_exec2(struct mos_linux_bo * bo,int used,struct mos_linux_context * ctx,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int flags,int * fence)2489 do_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
2490 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
2491 unsigned int flags, int *fence)
2492 {
2493 if(GetDrmMode())
2494 return 0; //libdrm_mock
2495
2496 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2497 struct drm_i915_gem_execbuffer2 execbuf;
2498 int ret = 0;
2499 int i;
2500
2501 if (to_bo_gem(bo)->has_error)
2502 return -ENOMEM;
2503
2504 switch (flags & 0x7) {
2505 default:
2506 return -EINVAL;
2507 case I915_EXEC_BLT:
2508 if (!bufmgr_gem->has_blt)
2509 return -EINVAL;
2510 break;
2511 case I915_EXEC_BSD:
2512 if (!bufmgr_gem->has_bsd)
2513 return -EINVAL;
2514 break;
2515 case I915_EXEC_VEBOX:
2516 if (!bufmgr_gem->has_vebox)
2517 return -EINVAL;
2518 break;
2519 case I915_EXEC_RENDER:
2520 case I915_EXEC_DEFAULT:
2521 break;
2522 }
2523
2524 pthread_mutex_lock(&bufmgr_gem->lock);
2525 /* Update indices and set up the validate list. */
2526 mos_gem_bo_process_reloc2(bo);
2527
2528 /* Add the batch buffer to the validation list. There are no relocations
2529 * pointing to it.
2530 */
2531 mos_add_validate_buffer2(bo, 0);
2532
2533 memclear(execbuf);
2534 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
2535 execbuf.buffer_count = bufmgr_gem->exec_count;
2536 execbuf.batch_start_offset = 0;
2537 execbuf.batch_len = used;
2538 execbuf.cliprects_ptr = (uintptr_t)cliprects;
2539 execbuf.num_cliprects = num_cliprects;
2540 execbuf.DR1 = 0;
2541 execbuf.DR4 = DR4;
2542 execbuf.flags = flags;
2543 if (ctx == nullptr)
2544 i915_execbuffer2_set_context_id(execbuf, 0);
2545 else
2546 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
2547 execbuf.rsvd2 = 0;
2548 if(flags & I915_EXEC_FENCE_SUBMIT)
2549 {
2550 execbuf.rsvd2 = *fence;
2551 }
2552 if(flags & I915_EXEC_FENCE_OUT)
2553 {
2554 execbuf.rsvd2 = -1;
2555 }
2556
2557 if (bufmgr_gem->no_exec)
2558 goto skip_execution;
2559
2560 ret = drmIoctl(bufmgr_gem->fd,
2561 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
2562 &execbuf);
2563 if (ret != 0) {
2564 ret = -errno;
2565 if (ret == -ENOSPC) {
2566 MOS_DBG("Execbuffer fails to pin. "
2567 "Estimate: %u. Actual: %u. Available: %u\n",
2568 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
2569 bufmgr_gem->exec_count),
2570 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
2571 bufmgr_gem->exec_count),
2572 (unsigned int) bufmgr_gem->gtt_size);
2573 }
2574 }
2575
2576 if (ctx != nullptr)
2577 {
2578 mos_update_buffer_offsets2(bufmgr_gem, ctx, bo);
2579 }
2580
2581 if(flags & I915_EXEC_FENCE_OUT)
2582 {
2583 *fence = execbuf.rsvd2 >> 32;
2584 }
2585
2586 skip_execution:
2587 if (bufmgr_gem->bufmgr.debug)
2588 mos_gem_dump_validation_list(bufmgr_gem);
2589
2590 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2591 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[i]);
2592
2593 bo_gem->idle = false;
2594
2595 /* Disconnect the buffer from the validate list */
2596 bo_gem->validate_index = -1;
2597 bufmgr_gem->exec_bos[i] = nullptr;
2598 }
2599 bufmgr_gem->exec_count = 0;
2600 pthread_mutex_unlock(&bufmgr_gem->lock);
2601
2602 return ret;
2603 }
2604
2605 static int
mos_gem_bo_exec2(struct mos_linux_bo * bo,int used,drm_clip_rect_t * cliprects,int num_cliprects,int DR4)2606 mos_gem_bo_exec2(struct mos_linux_bo *bo, int used,
2607 drm_clip_rect_t *cliprects, int num_cliprects,
2608 int DR4)
2609 {
2610 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
2611 I915_EXEC_RENDER, nullptr);
2612 }
2613
2614 static int
mos_gem_bo_mrb_exec2(struct mos_linux_bo * bo,int used,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int flags)2615 mos_gem_bo_mrb_exec2(struct mos_linux_bo *bo, int used,
2616 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
2617 unsigned int flags)
2618 {
2619 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
2620 flags, nullptr);
2621 }
2622
2623 static int
mos_gem_bo_context_exec2(struct mos_linux_bo * bo,int used,struct mos_linux_context * ctx,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int flags,int * fence)2624 mos_gem_bo_context_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
2625 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
2626 unsigned int flags,int *fence)
2627 {
2628 return do_exec2(bo, used, ctx, cliprects, num_cliprects, DR4,
2629 flags,fence);
2630 }
2631
2632 static int
mos_gem_bo_set_tiling_internal(struct mos_linux_bo * bo,uint32_t tiling_mode,uint32_t stride)2633 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
2634 uint32_t tiling_mode,
2635 uint32_t stride)
2636 {
2637 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2638 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2639 struct drm_i915_gem_set_tiling set_tiling;
2640 int ret;
2641
2642 if (bo_gem->global_name == 0 &&
2643 tiling_mode == bo_gem->tiling_mode &&
2644 stride == bo_gem->stride)
2645 return 0;
2646
2647 memset(&set_tiling, 0, sizeof(set_tiling));
2648 do {
2649 /* set_tiling is slightly broken and overwrites the
2650 * input on the error path, so we have to open code
2651 * rmIoctl.
2652 */
2653 set_tiling.handle = bo_gem->gem_handle;
2654 set_tiling.tiling_mode = tiling_mode;
2655 set_tiling.stride = stride;
2656
2657 ret = ioctl(bufmgr_gem->fd,
2658 DRM_IOCTL_I915_GEM_SET_TILING,
2659 &set_tiling);
2660 } while (ret == -1 && (errno == EINTR || errno == EAGAIN));
2661 if (ret == -1)
2662 return -errno;
2663
2664 bo_gem->tiling_mode = set_tiling.tiling_mode;
2665 bo_gem->swizzle_mode = set_tiling.swizzle_mode;
2666 bo_gem->stride = set_tiling.stride;
2667 return 0;
2668 }
2669
2670 static int
mos_gem_bo_set_tiling(struct mos_linux_bo * bo,uint32_t * tiling_mode,uint32_t stride)2671 mos_gem_bo_set_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
2672 uint32_t stride)
2673 {
2674 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2675 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2676 int ret;
2677
2678 /* Tiling with userptr surfaces is not supported
2679 * on all hardware so refuse it for time being.
2680 */
2681 if (bo_gem->is_userptr)
2682 return -EINVAL;
2683
2684 /* Linear buffers have no stride. By ensuring that we only ever use
2685 * stride 0 with linear buffers, we simplify our code.
2686 */
2687 if (*tiling_mode == I915_TILING_NONE)
2688 stride = 0;
2689
2690 ret = mos_gem_bo_set_tiling_internal(bo, *tiling_mode, stride);
2691 if (ret == 0)
2692 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
2693 *tiling_mode = bo_gem->tiling_mode;
2694 return ret;
2695 }
2696
2697 static int
mos_gem_bo_get_tiling(struct mos_linux_bo * bo,uint32_t * tiling_mode,uint32_t * swizzle_mode)2698 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
2699 uint32_t * swizzle_mode)
2700 {
2701 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2702
2703 *tiling_mode = bo_gem->tiling_mode;
2704 *swizzle_mode = bo_gem->swizzle_mode;
2705 return 0;
2706 }
2707
2708 static int
mos_gem_bo_set_softpin_offset(struct mos_linux_bo * bo,uint64_t offset)2709 mos_gem_bo_set_softpin_offset(struct mos_linux_bo *bo, uint64_t offset)
2710 {
2711 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2712
2713 bo_gem->is_softpin = true;
2714 bo->offset64 = offset;
2715 bo->offset = offset;
2716 return 0;
2717 }
2718
2719 static int
mos_gem_bo_set_softpin(MOS_LINUX_BO * bo)2720 mos_gem_bo_set_softpin(MOS_LINUX_BO *bo)
2721 {
2722 int ret = 0;
2723 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2724
2725 if (!mos_gem_bo_is_softpin(bo))
2726 {
2727 uint64_t offset = mos_gem_bo_vma_alloc(bo->bufmgr, MEMZONE_SYS, bo->size, PAGE_SIZE_64K);
2728 ret = mos_gem_bo_set_softpin_offset(bo, offset);
2729 }
2730
2731 if (ret == 0)
2732 {
2733 ret = mos_bo_use_48b_address_range(bo, 1);
2734 }
2735
2736 return ret;
2737 }
2738
2739 static struct mos_linux_bo *
mos_gem_bo_create_from_prime(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_prime * alloc_prime)2740 mos_gem_bo_create_from_prime(struct mos_bufmgr *bufmgr, struct mos_drm_bo_alloc_prime *alloc_prime)
2741 {
2742 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
2743 int ret;
2744 uint32_t handle;
2745 struct mos_bo_gem *bo_gem;
2746 struct drm_i915_gem_get_tiling get_tiling;
2747 int prime_fd = alloc_prime->prime_fd;
2748 int size = alloc_prime->size;
2749 drmMMListHead *list;
2750
2751 pthread_mutex_lock(&bufmgr_gem->lock);
2752 ret = drmPrimeFDToHandle(bufmgr_gem->fd, prime_fd, &handle);
2753 if (ret) {
2754 MOS_DBG("create_from_prime: failed to obtain handle from fd: %s\n", strerror(errno));
2755 pthread_mutex_unlock(&bufmgr_gem->lock);
2756 return nullptr;
2757 }
2758
2759 /*
2760 * See if the kernel has already returned this buffer to us. Just as
2761 * for named buffers, we must not create two bo's pointing at the same
2762 * kernel object
2763 */
2764 for (list = bufmgr_gem->named.next;
2765 list != &bufmgr_gem->named;
2766 list = list->next) {
2767 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
2768 if (bo_gem->gem_handle == handle) {
2769 mos_gem_bo_reference(&bo_gem->bo);
2770 pthread_mutex_unlock(&bufmgr_gem->lock);
2771 return &bo_gem->bo;
2772 }
2773 }
2774
2775 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
2776 if (!bo_gem) {
2777 pthread_mutex_unlock(&bufmgr_gem->lock);
2778 return nullptr;
2779 }
2780 /* Determine size of bo. The fd-to-handle ioctl really should
2781 * return the size, but it doesn't. If we have kernel 3.12 or
2782 * later, we can lseek on the prime fd to get the size. Older
2783 * kernels will just fail, in which case we fall back to the
2784 * provided (estimated or guess size). */
2785 ret = lseek(prime_fd, 0, SEEK_END);
2786 if (ret != -1)
2787 bo_gem->bo.size = ret;
2788 else
2789 bo_gem->bo.size = size;
2790
2791 bo_gem->bo.handle = handle;
2792 bo_gem->bo.bufmgr = bufmgr;
2793
2794 bo_gem->gem_handle = handle;
2795
2796 atomic_set(&bo_gem->refcount, 1);
2797
2798 bo_gem->name = alloc_prime->name;
2799 bo_gem->validate_index = -1;
2800 bo_gem->reloc_tree_fences = 0;
2801 bo_gem->used_as_reloc_target = false;
2802 bo_gem->has_error = false;
2803 bo_gem->reusable = false;
2804 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
2805
2806 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
2807 pthread_mutex_unlock(&bufmgr_gem->lock);
2808
2809 memclear(get_tiling);
2810 get_tiling.handle = bo_gem->gem_handle;
2811 ret = drmIoctl(bufmgr_gem->fd,
2812 DRM_IOCTL_I915_GEM_GET_TILING,
2813 &get_tiling);
2814 if (ret != 0) {
2815 MOS_DBG("create_from_prime: failed to get tiling: %s\n", strerror(errno));
2816 mos_gem_bo_unreference(&bo_gem->bo);
2817 return nullptr;
2818 }
2819 bo_gem->tiling_mode = get_tiling.tiling_mode;
2820 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
2821 /* XXX stride is unknown */
2822 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
2823 return &bo_gem->bo;
2824 }
2825
2826 static int
mos_gem_bo_export_to_prime(struct mos_linux_bo * bo,int * prime_fd)2827 mos_gem_bo_export_to_prime(struct mos_linux_bo *bo, int *prime_fd)
2828 {
2829 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2830 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2831
2832 pthread_mutex_lock(&bufmgr_gem->lock);
2833 if (DRMLISTEMPTY(&bo_gem->name_list))
2834 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
2835 pthread_mutex_unlock(&bufmgr_gem->lock);
2836
2837 if (drmPrimeHandleToFD(bufmgr_gem->fd, bo_gem->gem_handle,
2838 DRM_CLOEXEC, prime_fd) != 0)
2839 return -errno;
2840
2841 bo_gem->reusable = false;
2842
2843 return 0;
2844 }
2845
2846 static int
mos_gem_bo_flink(struct mos_linux_bo * bo,uint32_t * name)2847 mos_gem_bo_flink(struct mos_linux_bo *bo, uint32_t * name)
2848 {
2849 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2850 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2851 int ret;
2852
2853 if (!bo_gem->global_name) {
2854 struct drm_gem_flink flink;
2855
2856 memclear(flink);
2857 flink.handle = bo_gem->gem_handle;
2858
2859 pthread_mutex_lock(&bufmgr_gem->lock);
2860
2861 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
2862 if (ret != 0) {
2863 pthread_mutex_unlock(&bufmgr_gem->lock);
2864 return -errno;
2865 }
2866
2867 bo_gem->global_name = flink.name;
2868 bo_gem->reusable = false;
2869
2870 if (DRMLISTEMPTY(&bo_gem->name_list))
2871 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
2872 pthread_mutex_unlock(&bufmgr_gem->lock);
2873 }
2874
2875 *name = bo_gem->global_name;
2876 return 0;
2877 }
2878
2879 /**
2880 * Enables unlimited caching of buffer objects for reuse.
2881 *
2882 * This is potentially very memory expensive, as the cache at each bucket
2883 * size is only bounded by how many buffers of that size we've managed to have
2884 * in flight at once.
2885 */
2886 static void
mos_gem_enable_reuse(struct mos_bufmgr * bufmgr)2887 mos_gem_enable_reuse(struct mos_bufmgr *bufmgr)
2888 {
2889 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
2890
2891 bufmgr_gem->bo_reuse = true;
2892 }
2893
2894 /**
2895 * Return the additional aperture space required by the tree of buffer objects
2896 * rooted at bo.
2897 */
2898 static int
mos_gem_bo_get_aperture_space(struct mos_linux_bo * bo)2899 mos_gem_bo_get_aperture_space(struct mos_linux_bo *bo)
2900 {
2901 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2902 int i;
2903 int total = 0;
2904
2905 if (bo == nullptr || bo_gem->included_in_check_aperture)
2906 return 0;
2907
2908 total += bo->size;
2909 bo_gem->included_in_check_aperture = true;
2910
2911 for (i = 0; i < bo_gem->reloc_count; i++)
2912 total +=
2913 mos_gem_bo_get_aperture_space(bo_gem->
2914 reloc_target_info[i].bo);
2915
2916 return total;
2917 }
2918
2919 /**
2920 * Count the number of buffers in this list that need a fence reg
2921 *
2922 * If the count is greater than the number of available regs, we'll have
2923 * to ask the caller to resubmit a batch with fewer tiled buffers.
2924 *
2925 * This function over-counts if the same buffer is used multiple times.
2926 */
2927 static unsigned int
mos_gem_total_fences(struct mos_linux_bo ** bo_array,int count)2928 mos_gem_total_fences(struct mos_linux_bo ** bo_array, int count)
2929 {
2930 int i;
2931 unsigned int total = 0;
2932
2933 for (i = 0; i < count; i++) {
2934 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
2935
2936 if (bo_gem == nullptr)
2937 continue;
2938
2939 total += bo_gem->reloc_tree_fences;
2940 }
2941 return total;
2942 }
2943
2944 /**
2945 * Clear the flag set by drm_intel_gem_bo_get_aperture_space() so we're ready
2946 * for the next drm_intel_bufmgr_check_aperture_space() call.
2947 */
2948 static void
mos_gem_bo_clear_aperture_space_flag(struct mos_linux_bo * bo)2949 mos_gem_bo_clear_aperture_space_flag(struct mos_linux_bo *bo)
2950 {
2951 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2952 int i;
2953
2954 if (bo == nullptr || !bo_gem->included_in_check_aperture)
2955 return;
2956
2957 bo_gem->included_in_check_aperture = false;
2958
2959 for (i = 0; i < bo_gem->reloc_count; i++)
2960 mos_gem_bo_clear_aperture_space_flag(bo_gem->
2961 reloc_target_info[i].bo);
2962 }
2963
2964 /**
2965 * Return a conservative estimate for the amount of aperture required
2966 * for a collection of buffers. This may double-count some buffers.
2967 */
2968 static unsigned int
mos_gem_estimate_batch_space(struct mos_linux_bo ** bo_array,int count)2969 mos_gem_estimate_batch_space(struct mos_linux_bo **bo_array, int count)
2970 {
2971 int i;
2972 unsigned int total = 0;
2973
2974 for (i = 0; i < count; i++) {
2975 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
2976 if (bo_gem != nullptr)
2977 total += bo_gem->reloc_tree_size;
2978 }
2979 return total;
2980 }
2981
2982 /**
2983 * Return the amount of aperture needed for a collection of buffers.
2984 * This avoids double counting any buffers, at the cost of looking
2985 * at every buffer in the set.
2986 */
2987 static unsigned int
mos_gem_compute_batch_space(struct mos_linux_bo ** bo_array,int count)2988 mos_gem_compute_batch_space(struct mos_linux_bo **bo_array, int count)
2989 {
2990 int i;
2991 unsigned int total = 0;
2992
2993 for (i = 0; i < count; i++) {
2994 total += mos_gem_bo_get_aperture_space(bo_array[i]);
2995 /* For the first buffer object in the array, we get an
2996 * accurate count back for its reloc_tree size (since nothing
2997 * had been flagged as being counted yet). We can save that
2998 * value out as a more conservative reloc_tree_size that
2999 * avoids double-counting target buffers. Since the first
3000 * buffer happens to usually be the batch buffer in our
3001 * callers, this can pull us back from doing the tree
3002 * walk on every new batch emit.
3003 */
3004 if (i == 0) {
3005 struct mos_bo_gem *bo_gem =
3006 (struct mos_bo_gem *) bo_array[i];
3007 bo_gem->reloc_tree_size = total;
3008 }
3009 }
3010
3011 for (i = 0; i < count; i++)
3012 mos_gem_bo_clear_aperture_space_flag(bo_array[i]);
3013 return total;
3014 }
3015
3016 /**
3017 * Return -1 if the batchbuffer should be flushed before attempting to
3018 * emit rendering referencing the buffers pointed to by bo_array.
3019 *
3020 * This is required because if we try to emit a batchbuffer with relocations
3021 * to a tree of buffers that won't simultaneously fit in the aperture,
3022 * the rendering will return an error at a point where the software is not
3023 * prepared to recover from it.
3024 *
3025 * However, we also want to emit the batchbuffer significantly before we reach
3026 * the limit, as a series of batchbuffers each of which references buffers
3027 * covering almost all of the aperture means that at each emit we end up
3028 * waiting to evict a buffer from the last rendering, and we get synchronous
3029 * performance. By emitting smaller batchbuffers, we eat some CPU overhead to
3030 * get better parallelism.
3031 */
3032 static int
mos_gem_check_aperture_space(struct mos_linux_bo ** bo_array,int count)3033 mos_gem_check_aperture_space(struct mos_linux_bo **bo_array, int count)
3034 {
3035 struct mos_bufmgr_gem *bufmgr_gem =
3036 (struct mos_bufmgr_gem *) bo_array[0]->bufmgr;
3037 unsigned int total = 0;
3038 unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
3039 int total_fences;
3040
3041 /* Check for fence reg constraints if necessary */
3042 if (bufmgr_gem->available_fences) {
3043 total_fences = mos_gem_total_fences(bo_array, count);
3044 if (total_fences > bufmgr_gem->available_fences)
3045 return -ENOSPC;
3046 }
3047
3048 total = mos_gem_estimate_batch_space(bo_array, count);
3049
3050 if (total > threshold)
3051 total = mos_gem_compute_batch_space(bo_array, count);
3052
3053 if (total > threshold) {
3054 MOS_DBG("check_space: overflowed available aperture, "
3055 "%dkb vs %dkb\n",
3056 total / 1024, (int)bufmgr_gem->gtt_size / 1024);
3057 return -ENOSPC;
3058 } else {
3059 MOS_DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024,
3060 (int)bufmgr_gem->gtt_size / 1024);
3061 return 0;
3062 }
3063 }
3064
3065 /*
3066 * Disable buffer reuse for objects which are shared with the kernel
3067 * as scanout buffers
3068 */
3069 static int
mos_gem_bo_disable_reuse(struct mos_linux_bo * bo)3070 mos_gem_bo_disable_reuse(struct mos_linux_bo *bo)
3071 {
3072 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3073
3074 bo_gem->reusable = false;
3075 return 0;
3076 }
3077
3078 static int
mos_gem_bo_is_reusable(struct mos_linux_bo * bo)3079 mos_gem_bo_is_reusable(struct mos_linux_bo *bo)
3080 {
3081 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3082
3083 return bo_gem->reusable;
3084 }
3085
3086 static int
_mos_gem_bo_references(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo)3087 _mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3088 {
3089 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3090 int i;
3091
3092 for (i = 0; i < bo_gem->reloc_count; i++) {
3093 if (bo_gem->reloc_target_info[i].bo == target_bo)
3094 return 1;
3095 if (bo == bo_gem->reloc_target_info[i].bo)
3096 continue;
3097 if (_mos_gem_bo_references(bo_gem->reloc_target_info[i].bo,
3098 target_bo))
3099 return 1;
3100 }
3101
3102 for (i = 0; i< bo_gem->softpin_target_count; i++) {
3103 if (bo_gem->softpin_target[i].bo == target_bo)
3104 return 1;
3105 if (_mos_gem_bo_references(bo_gem->softpin_target[i].bo, target_bo))
3106 return 1;
3107 }
3108
3109 return 0;
3110 }
3111
3112 /** Return true if target_bo is referenced by bo's relocation tree. */
3113 static int
mos_gem_bo_references(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo)3114 mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3115 {
3116 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
3117
3118 if (bo == nullptr || target_bo == nullptr)
3119 return 0;
3120 if (target_bo_gem->used_as_reloc_target)
3121 return _mos_gem_bo_references(bo, target_bo);
3122 return 0;
3123 }
3124
3125 static void
add_bucket(struct mos_bufmgr_gem * bufmgr_gem,int size)3126 add_bucket(struct mos_bufmgr_gem *bufmgr_gem, int size)
3127 {
3128 unsigned int i = bufmgr_gem->num_buckets;
3129
3130 assert(i < ARRAY_SIZE(bufmgr_gem->cache_bucket));
3131
3132 DRMINITLISTHEAD(&bufmgr_gem->cache_bucket[i].head);
3133 bufmgr_gem->cache_bucket[i].size = size;
3134 bufmgr_gem->num_buckets++;
3135 }
3136
3137 static void
init_cache_buckets(struct mos_bufmgr_gem * bufmgr_gem)3138 init_cache_buckets(struct mos_bufmgr_gem *bufmgr_gem)
3139 {
3140 unsigned long size, cache_max_size = 64 * 1024 * 1024;
3141
3142 /* OK, so power of two buckets was too wasteful of memory.
3143 * Give 3 other sizes between each power of two, to hopefully
3144 * cover things accurately enough. (The alternative is
3145 * probably to just go for exact matching of sizes, and assume
3146 * that for things like composited window resize the tiled
3147 * width/height alignment and rounding of sizes to pages will
3148 * get us useful cache hit rates anyway)
3149 */
3150 add_bucket(bufmgr_gem, 4096);
3151 add_bucket(bufmgr_gem, 4096 * 2);
3152 add_bucket(bufmgr_gem, 4096 * 3);
3153
3154 /* Initialize the linked lists for BO reuse cache. */
3155 for (size = 4 * 4096; size <= cache_max_size; size *= 2) {
3156 add_bucket(bufmgr_gem, size);
3157
3158 add_bucket(bufmgr_gem, size + size * 1 / 4);
3159 add_bucket(bufmgr_gem, size + size * 2 / 4);
3160 add_bucket(bufmgr_gem, size + size * 3 / 4);
3161 }
3162 }
3163
3164 /**
3165 * Get the PCI ID for the device. This can be overridden by setting the
3166 * INTEL_DEVID_OVERRIDE environment variable to the desired ID.
3167 */
3168 static int
get_pci_device_id(struct mos_bufmgr_gem * bufmgr_gem)3169 get_pci_device_id(struct mos_bufmgr_gem *bufmgr_gem)
3170 {
3171 char *devid_override;
3172 int devid = 0;
3173 int ret;
3174 drm_i915_getparam_t gp;
3175
3176 if (geteuid() == getuid()) {
3177 devid_override = getenv("INTEL_DEVID_OVERRIDE");
3178 if (devid_override) {
3179 bufmgr_gem->no_exec = true;
3180 return strtod(devid_override, nullptr);
3181 }
3182 }
3183
3184 memclear(gp);
3185 gp.param = I915_PARAM_CHIPSET_ID;
3186 gp.value = &devid;
3187 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3188 if (ret) {
3189 fprintf(stderr, "get chip id failed: %d [%d]\n", ret, errno);
3190 fprintf(stderr, "param: %d, val: %d\n", gp.param, *gp.value);
3191 }
3192 return devid;
3193 }
3194
3195 static int
mos_gem_get_devid(struct mos_bufmgr * bufmgr)3196 mos_gem_get_devid(struct mos_bufmgr *bufmgr)
3197 {
3198 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3199
3200 return bufmgr_gem->pci_device;
3201 }
3202
3203 static struct mos_linux_context *
mos_gem_context_create(struct mos_bufmgr * bufmgr)3204 mos_gem_context_create(struct mos_bufmgr *bufmgr)
3205 {
3206 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3207 struct drm_i915_gem_context_create create;
3208 struct mos_linux_context *context = nullptr;
3209 int ret;
3210
3211 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
3212 if (!context)
3213 return nullptr;
3214
3215 memclear(create);
3216 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
3217 if (ret != 0) {
3218 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
3219 strerror(errno));
3220 free(context);
3221 return nullptr;
3222 }
3223
3224 context->ctx_id = create.ctx_id;
3225 context->bufmgr = bufmgr;
3226
3227 return context;
3228 }
3229
3230 static void
mos_gem_context_destroy(struct mos_linux_context * ctx)3231 mos_gem_context_destroy(struct mos_linux_context *ctx)
3232 {
3233 struct mos_bufmgr_gem *bufmgr_gem;
3234 struct drm_i915_gem_context_destroy destroy;
3235 int ret;
3236
3237 if (ctx == nullptr)
3238 return;
3239
3240 memclear(destroy);
3241
3242 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
3243 destroy.ctx_id = ctx->ctx_id;
3244 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY,
3245 &destroy);
3246 if (ret != 0)
3247 fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
3248 strerror(errno));
3249
3250 free(ctx);
3251 }
3252
3253 static int
mos_bufmg_get_reset_stats(struct mos_linux_context * ctx,uint32_t * reset_count,uint32_t * active,uint32_t * pending)3254 mos_bufmg_get_reset_stats(struct mos_linux_context *ctx,
3255 uint32_t *reset_count,
3256 uint32_t *active,
3257 uint32_t *pending)
3258 {
3259 struct mos_bufmgr_gem *bufmgr_gem;
3260 struct drm_i915_reset_stats stats;
3261 int ret;
3262
3263 if (ctx == nullptr)
3264 return -EINVAL;
3265
3266 memclear(stats);
3267
3268 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
3269 stats.ctx_id = ctx->ctx_id;
3270 ret = drmIoctl(bufmgr_gem->fd,
3271 DRM_IOCTL_I915_GET_RESET_STATS,
3272 &stats);
3273 if (ret == 0) {
3274 if (reset_count != nullptr)
3275 *reset_count = stats.reset_count;
3276
3277 if (active != nullptr)
3278 *active = stats.batch_active;
3279
3280 if (pending != nullptr)
3281 *pending = stats.batch_pending;
3282 }
3283
3284 return ret;
3285 }
3286
mos_bufmgr_hweight8(struct mos_linux_context * ctx,uint8_t w)3287 static unsigned int mos_bufmgr_hweight8(struct mos_linux_context *ctx, uint8_t w)
3288 {
3289 uint32_t i, weight = 0;
3290
3291 for (i=0; i<8; i++)
3292 {
3293 weight += !!((w) & (1UL << i));
3294 }
3295 return weight;
3296 }
3297
mos_bufmgr_switch_off_n_bits(struct mos_linux_context * ctx,uint8_t in_mask,int n)3298 static uint8_t mos_bufmgr_switch_off_n_bits(struct mos_linux_context *ctx, uint8_t in_mask, int n)
3299 {
3300 int i,count;
3301 uint8_t bi,out_mask;
3302
3303 assert (n>0 && n<=8);
3304
3305 out_mask = in_mask;
3306 count = n;
3307 for(i=0; i<8; i++)
3308 {
3309 bi = 1UL<<i;
3310 if (bi & in_mask)
3311 {
3312 out_mask &= ~bi;
3313 count--;
3314 }
3315 if (count==0)
3316 {
3317 break;
3318 }
3319 }
3320 return out_mask;
3321 }
3322
3323 static int
mos_bufmgr_get_context_param_sseu(struct mos_linux_context * ctx,struct drm_i915_gem_context_param_sseu * sseu)3324 mos_bufmgr_get_context_param_sseu(struct mos_linux_context *ctx,
3325 struct drm_i915_gem_context_param_sseu *sseu)
3326 {
3327 struct mos_bufmgr_gem *bufmgr_gem;
3328 struct drm_i915_gem_context_param context_param;
3329 int ret;
3330
3331 if (ctx == nullptr)
3332 return -EINVAL;
3333
3334 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
3335 memset(&context_param, 0, sizeof(context_param));
3336 context_param.ctx_id = ctx->ctx_id;
3337 context_param.param = I915_CONTEXT_PARAM_SSEU;
3338 context_param.value = (uint64_t) sseu;
3339 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
3340
3341 ret = drmIoctl(bufmgr_gem->fd,
3342 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
3343 &context_param);
3344
3345 return ret;
3346 }
3347
3348 static int
mos_bufmgr_set_context_param_sseu(struct mos_linux_context * ctx,struct drm_i915_gem_context_param_sseu sseu)3349 mos_bufmgr_set_context_param_sseu(struct mos_linux_context *ctx,
3350 struct drm_i915_gem_context_param_sseu sseu)
3351 {
3352 struct mos_bufmgr_gem *bufmgr_gem;
3353 struct drm_i915_gem_context_param context_param;
3354 int ret;
3355
3356 if (ctx == nullptr)
3357 return -EINVAL;
3358
3359 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
3360 memset(&context_param, 0, sizeof(context_param));
3361 context_param.ctx_id = ctx->ctx_id;
3362 context_param.param = I915_CONTEXT_PARAM_SSEU;
3363 context_param.value = (uint64_t) &sseu;
3364 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
3365
3366 ret = drmIoctl(bufmgr_gem->fd,
3367 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
3368 &context_param);
3369
3370 return ret;
3371 }
3372
3373 static int
mos_gem_get_context_param(struct mos_linux_context * ctx,uint32_t size,uint64_t param,uint64_t * value)3374 mos_gem_get_context_param(struct mos_linux_context *ctx,
3375 uint32_t size,
3376 uint64_t param,
3377 uint64_t *value)
3378 {
3379 struct mos_bufmgr_gem *bufmgr_gem;
3380 struct drm_i915_gem_context_param context_param;
3381 int ret;
3382
3383 if (ctx == nullptr)
3384 return -EINVAL;
3385
3386 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
3387 context_param.ctx_id = ctx->ctx_id;
3388 context_param.size = size;
3389 context_param.param = param;
3390 context_param.value = 0;
3391
3392 ret = drmIoctl(bufmgr_gem->fd,
3393 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
3394 &context_param);
3395 *value = context_param.value;
3396
3397 return ret;
3398 }
3399
3400 static int
mos_gem_set_context_param(struct mos_linux_context * ctx,uint32_t size,uint64_t param,uint64_t value)3401 mos_gem_set_context_param(struct mos_linux_context *ctx,
3402 uint32_t size,
3403 uint64_t param,
3404 uint64_t value)
3405 {
3406 struct mos_bufmgr_gem *bufmgr_gem;
3407 struct drm_i915_gem_context_param context_param;
3408 int ret;
3409
3410 if (ctx == nullptr)
3411 return -EINVAL;
3412
3413 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
3414 context_param.ctx_id = ctx->ctx_id;
3415 context_param.size = size;
3416 context_param.param = param;
3417 context_param.value = value;
3418
3419 ret = drmIoctl(bufmgr_gem->fd,
3420 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
3421 &context_param);
3422
3423 return ret;
3424 }
3425
3426 static int
mos_bufmg_reg_read(struct mos_bufmgr * bufmgr,uint32_t offset,uint64_t * result)3427 mos_bufmg_reg_read(struct mos_bufmgr *bufmgr,
3428 uint32_t offset,
3429 uint64_t *result)
3430 {
3431 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3432 struct drm_i915_reg_read reg_read;
3433 int ret;
3434
3435 memclear(reg_read);
3436 reg_read.offset = offset;
3437
3438 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_REG_READ, ®_read);
3439
3440 *result = reg_read.val;
3441 return ret;
3442 }
3443
3444 static pthread_mutex_t bufmgr_list_mutex = PTHREAD_MUTEX_INITIALIZER;
3445 static drmMMListHead bufmgr_list = { &bufmgr_list, &bufmgr_list };
3446
3447 static struct mos_bufmgr_gem *
mos_bufmgr_gem_find(int fd)3448 mos_bufmgr_gem_find(int fd)
3449 {
3450 struct mos_bufmgr_gem *bufmgr_gem;
3451
3452 DRMLISTFOREACHENTRY(bufmgr_gem, &bufmgr_list, managers) {
3453 if (bufmgr_gem->fd == fd) {
3454 atomic_inc(&bufmgr_gem->refcount);
3455 return bufmgr_gem;
3456 }
3457 }
3458
3459 return nullptr;
3460 }
3461
3462 static void
mos_bufmgr_gem_unref(struct mos_bufmgr * bufmgr)3463 mos_bufmgr_gem_unref(struct mos_bufmgr *bufmgr)
3464 {
3465 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3466
3467 if (atomic_add_unless(&bufmgr_gem->refcount, -1, 1)) {
3468 pthread_mutex_lock(&bufmgr_list_mutex);
3469
3470 if (atomic_dec_and_test(&bufmgr_gem->refcount)) {
3471 DRMLISTDEL(&bufmgr_gem->managers);
3472 mos_bufmgr_gem_destroy(bufmgr);
3473 }
3474
3475 pthread_mutex_unlock(&bufmgr_list_mutex);
3476 }
3477 }
3478
3479 static struct mos_linux_context *
mos_gem_context_create_ext(struct mos_bufmgr * bufmgr,__u32 flags,bool bContextProtected)3480 mos_gem_context_create_ext(struct mos_bufmgr *bufmgr, __u32 flags, bool bContextProtected)
3481 {
3482 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3483 struct drm_i915_gem_context_create_ext create;
3484 struct mos_linux_context *context = nullptr;
3485 struct drm_i915_gem_context_create_ext_setparam p_protected;
3486 struct drm_i915_gem_context_create_ext_setparam p_norecover;
3487 int ret;
3488
3489 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
3490 if (!context)
3491 return nullptr;
3492
3493 memclear(create);
3494 create.flags = flags;
3495 create.extensions = 0;
3496 if (bContextProtected)
3497 {
3498 memclear(p_protected);
3499 memclear(p_norecover);
3500 p_protected.base.next_extension = 0;
3501 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
3502 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
3503 p_protected.param.value = 1;
3504
3505 p_norecover.base.next_extension = (uintptr_t)&p_protected;
3506 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
3507 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
3508 p_norecover.param.value = 0;
3509
3510 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
3511 create.extensions = (uintptr_t)&p_norecover;
3512 }
3513 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
3514 if (ret != 0) {
3515 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
3516 strerror(errno));
3517 free(context);
3518 return nullptr;
3519 }
3520
3521 context->ctx_id = create.ctx_id;
3522 context->bufmgr = bufmgr;
3523
3524 return context;
3525 }
3526
mos_gem_vm_create(struct mos_bufmgr * bufmgr)3527 static __u32 mos_gem_vm_create(struct mos_bufmgr *bufmgr)
3528 {
3529 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3530 struct drm_i915_gem_vm_control *vm = nullptr;
3531 __u32 vm_id;
3532 int ret;
3533
3534 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
3535 if (nullptr == vm)
3536 {
3537 MOS_DBG("vm calloc failed\n" );
3538 return INVALID_VM;
3539 }
3540
3541 memset(vm, 0, sizeof(*vm));
3542
3543 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_CREATE, vm);
3544 if (ret != 0) {
3545 MOS_DBG("DRM_IOCTL_I915_GEM_VM_CREATE failed: %s\n",
3546 strerror(errno));
3547 free(vm);
3548 return INVALID_VM;
3549 }
3550
3551 vm_id = vm->vm_id;
3552 free(vm);
3553
3554 return vm_id;
3555 }
3556
3557
mos_gem_vm_destroy(struct mos_bufmgr * bufmgr,__u32 vm_id)3558 static void mos_gem_vm_destroy(struct mos_bufmgr *bufmgr, __u32 vm_id)
3559 {
3560 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3561 struct drm_i915_gem_vm_control *vm = nullptr;
3562 int ret;
3563
3564 if (vm_id == INVALID_VM)
3565 {
3566 MOS_DBG("input invalid param\n" );
3567 return;
3568 }
3569
3570 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
3571
3572 if (nullptr == vm)
3573 {
3574 MOS_DBG("vm calloc failed\n" );
3575 return;
3576 }
3577
3578 vm->vm_id = vm_id;
3579 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_DESTROY, vm);
3580 if (ret != 0) {
3581 MOS_DBG("DRM_IOCTL_I915_GEM_VM_DESTROY failed: %s\n",
3582 strerror(errno));
3583 }
3584
3585 free(vm);
3586 }
3587
3588 static struct mos_linux_context *
mos_gem_context_create_shared(struct mos_bufmgr * bufmgr,mos_linux_context * ctx,__u32 flags,bool bContextProtected,void * engine_map,uint8_t ctx_width,uint8_t num_placements,uint32_t ctx_type)3589 mos_gem_context_create_shared(struct mos_bufmgr *bufmgr,
3590 mos_linux_context* ctx,
3591 __u32 flags,
3592 bool bContextProtected,
3593 void *engine_map,
3594 uint8_t ctx_width,
3595 uint8_t num_placements,
3596 uint32_t ctx_type)
3597 {
3598 MOS_UNUSED(engine_map);
3599 MOS_UNUSED(ctx_width);
3600 MOS_UNUSED(num_placements);
3601 MOS_UNUSED(ctx_type);
3602
3603 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3604 struct drm_i915_gem_context_create_ext create;
3605 struct mos_linux_context *context = nullptr;
3606 struct drm_i915_gem_context_create_ext_setparam p_protected;
3607 struct drm_i915_gem_context_create_ext_setparam p_norecover;
3608 int ret;
3609
3610 if (ctx == nullptr || ctx->vm_id == INVALID_VM)
3611 return nullptr;
3612
3613 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
3614 if (!context)
3615 return nullptr;
3616
3617 memclear(create);
3618 create.flags = flags;
3619 create.extensions = 0;
3620 if (bContextProtected)
3621 {
3622 memclear(p_protected);
3623 memclear(p_norecover);
3624 p_protected.base.next_extension = 0;
3625 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
3626 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
3627 p_protected.param.value = 1;
3628
3629 p_norecover.base.next_extension = (uintptr_t)&p_protected;
3630 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
3631 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
3632 p_norecover.param.value = 0;
3633
3634 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
3635 create.extensions = (uintptr_t)&p_norecover;
3636 }
3637 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
3638 if (ret != 0) {
3639 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
3640 strerror(errno));
3641 free(context);
3642 return nullptr;
3643 }
3644
3645 context->ctx_id = create.ctx_id;
3646 context->bufmgr = bufmgr;
3647
3648 ret = mos_set_context_param(context,
3649 0,
3650 I915_CONTEXT_PARAM_VM,
3651 ctx->vm_id);
3652 if(ret != 0) {
3653 MOS_DBG("I915_CONTEXT_PARAM_VM failed: %s\n",
3654 strerror(errno));
3655 free(context);
3656 return nullptr;
3657 }
3658
3659 return context;
3660 }
3661
mos_bufmgr_query_engines(struct mos_bufmgr * bufmgr,__u16 engine_class,__u64 caps,unsigned int * nengine,void * engine_map)3662 static int mos_bufmgr_query_engines(struct mos_bufmgr *bufmgr,
3663 __u16 engine_class,
3664 __u64 caps,
3665 unsigned int *nengine,
3666 void *engine_map)
3667 {
3668 struct i915_engine_class_instance *ci = (struct i915_engine_class_instance *)engine_map;
3669 if((bufmgr == nullptr) || (nengine == nullptr) || (ci == nullptr))
3670 {
3671 return -EINVAL;
3672 }
3673
3674 struct drm_i915_query query;
3675 struct drm_i915_query_item query_item;
3676 struct drm_i915_query_engine_info *engines = nullptr;
3677 int ret, len;
3678 int fd = ((struct mos_bufmgr_gem*)bufmgr)->fd;
3679
3680 memclear(query_item);
3681 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
3682 query_item.length = 0;
3683 memclear(query);
3684 query.num_items = 1;
3685 query.items_ptr = (uintptr_t)&query_item;
3686
3687 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
3688 if (ret)
3689 {
3690 goto fini;
3691 }
3692 len = query_item.length;
3693
3694 engines = (drm_i915_query_engine_info *)malloc(len);
3695 if (nullptr == engines)
3696 {
3697 ret = -ENOMEM;
3698 goto fini;
3699 }
3700 memset(engines,0,len);
3701 memclear(query_item);
3702 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
3703 query_item.length = len;
3704 query_item.data_ptr = (uintptr_t)engines;
3705 memclear(query);
3706 query.num_items = 1;
3707 query.items_ptr = (uintptr_t)&query_item;
3708
3709 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
3710 if (ret)
3711 {
3712 goto fini;
3713 }
3714
3715 int i, num;
3716 for (i = 0, num = 0; i < engines->num_engines; i++) {
3717 struct drm_i915_engine_info *engine =
3718 (struct drm_i915_engine_info *)&engines->engines[i];
3719 if ( engine_class == engine->engine.engine_class
3720 && ((caps & engine->capabilities) == caps ))
3721 {
3722 ci->engine_class = engine_class;
3723 ci->engine_instance = engine->engine.engine_instance;
3724 ci++;
3725 num++;
3726 }
3727 if (num > *nengine)
3728 {
3729 fprintf(stderr,"%s: Number of engine instances out of range, %d,%d\n",
3730 __FUNCTION__, num, *nengine);
3731 goto fini;
3732 }
3733 }
3734 *nengine = num;
3735
3736
3737 fini:
3738 if (engines)
3739 free(engines);
3740 return ret;
3741 }
3742
mos_gem_set_context_param_load_balance(struct mos_linux_context * ctx,struct i915_engine_class_instance * ci,unsigned int count)3743 static int mos_gem_set_context_param_load_balance(struct mos_linux_context *ctx,
3744 struct i915_engine_class_instance *ci,
3745 unsigned int count)
3746 {
3747 int ret;
3748 uint32_t size;
3749 struct i915_context_engines_load_balance* balancer = nullptr;
3750 struct i915_context_param_engines* set_engines = nullptr;
3751
3752 assert(ci);
3753
3754 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
3755 size = sizeof(struct i915_context_engines_load_balance) + count * sizeof(*ci);
3756 balancer = (struct i915_context_engines_load_balance*)malloc(size);
3757 if (NULL == balancer)
3758 {
3759 ret = -ENOMEM;
3760 goto fini;
3761 }
3762 memset(balancer, 0, size);
3763 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
3764 balancer->num_siblings = count;
3765 memcpy(balancer->engines, ci, count * sizeof(*ci));
3766
3767 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
3768 size = sizeof(uint64_t) + sizeof(*ci);
3769 set_engines = (struct i915_context_param_engines*) malloc(size);
3770 if (NULL == set_engines)
3771 {
3772 ret = -ENOMEM;
3773 goto fini;
3774 }
3775 set_engines->extensions = (uintptr_t)(balancer);
3776 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
3777 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
3778
3779 ret = mos_set_context_param(ctx,
3780 size,
3781 I915_CONTEXT_PARAM_ENGINES,
3782 (uintptr_t)set_engines);
3783 fini:
3784 if (set_engines)
3785 free(set_engines);
3786 if (balancer)
3787 free(balancer);
3788 return ret;
3789 }
3790
mos_gem_bo_is_softpin(struct mos_linux_bo * bo)3791 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo)
3792 {
3793 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3794 if (bo_gem == nullptr)
3795 {
3796 return false;
3797 }
3798
3799 return bo_gem->is_softpin;
3800 }
3801
mos_bufmgr_get_platform_information(struct mos_bufmgr * bufmgr)3802 static uint64_t mos_bufmgr_get_platform_information(struct mos_bufmgr *bufmgr)
3803 {
3804 assert(bufmgr);
3805 return bufmgr->platform_information;
3806 }
3807
mos_bufmgr_set_platform_information(struct mos_bufmgr * bufmgr,uint64_t p)3808 static void mos_bufmgr_set_platform_information(struct mos_bufmgr *bufmgr, uint64_t p)
3809 {
3810 assert(bufmgr);
3811 bufmgr->platform_information |= p;
3812 }
3813
3814 /**
3815 * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
3816 * and manage map buffer objections.
3817 *
3818 * \param fd File descriptor of the opened DRM device.
3819 */
3820 struct mos_bufmgr *
mos_bufmgr_gem_init(int fd,int batch_size,int * device_type)3821 mos_bufmgr_gem_init(int fd, int batch_size, int *device_type)
3822 {
3823 struct mos_bufmgr_gem *bufmgr_gem;
3824 struct drm_i915_gem_get_aperture aperture;
3825 drm_i915_getparam_t gp;
3826 int ret, tmp;
3827 bool exec2 = false;
3828 if (device_type != nullptr)
3829 {
3830 *device_type = DEVICE_TYPE_I915;
3831 }
3832 pthread_mutex_lock(&bufmgr_list_mutex);
3833
3834 bufmgr_gem = mos_bufmgr_gem_find(fd);
3835 if (bufmgr_gem)
3836 goto exit;
3837
3838 bufmgr_gem = (struct mos_bufmgr_gem *)calloc(1, sizeof(*bufmgr_gem));
3839 if (bufmgr_gem == nullptr)
3840 goto exit;
3841
3842 bufmgr_gem->fd = fd;
3843 atomic_set(&bufmgr_gem->refcount, 1);
3844
3845 if (pthread_mutex_init(&bufmgr_gem->lock, nullptr) != 0) {
3846 free(bufmgr_gem);
3847 bufmgr_gem = nullptr;
3848 goto exit;
3849 }
3850
3851 bufmgr_gem->bufmgr.bo_alloc = mos_gem_bo_alloc;
3852 bufmgr_gem->bufmgr.bo_alloc_tiled = mos_gem_bo_alloc_tiled;
3853 bufmgr_gem->bufmgr.bo_reference = mos_gem_bo_reference;
3854 bufmgr_gem->bufmgr.bo_unreference = mos_gem_bo_unreference;
3855 bufmgr_gem->bufmgr.bo_map = mos_gem_bo_map;
3856 bufmgr_gem->bufmgr.bo_unmap = mos_gem_bo_unmap;
3857 bufmgr_gem->bufmgr.bo_wait_rendering = mos_gem_bo_wait_rendering;
3858 bufmgr_gem->bufmgr.bo_pad_to_size = mos_gem_bo_pad_to_size;
3859 bufmgr_gem->bufmgr.bo_emit_reloc = mos_gem_bo_emit_reloc;
3860 bufmgr_gem->bufmgr.bo_get_tiling = mos_gem_bo_get_tiling;
3861 bufmgr_gem->bufmgr.bo_set_tiling = mos_gem_bo_set_tiling;
3862 bufmgr_gem->bufmgr.bo_flink = mos_gem_bo_flink;
3863 bufmgr_gem->bufmgr.bo_exec = mos_gem_bo_exec2;
3864 bufmgr_gem->bufmgr.bo_mrb_exec = mos_gem_bo_mrb_exec2;
3865 bufmgr_gem->bufmgr.bo_busy = mos_gem_bo_busy;
3866 bufmgr_gem->bufmgr.bo_madvise = mos_gem_bo_madvise;
3867 bufmgr_gem->bufmgr.destroy = mos_bufmgr_gem_unref;
3868 bufmgr_gem->bufmgr.debug = 0;
3869 bufmgr_gem->bufmgr.check_aperture_space =
3870 mos_gem_check_aperture_space;
3871 bufmgr_gem->bufmgr.bo_disable_reuse = mos_gem_bo_disable_reuse;
3872 bufmgr_gem->bufmgr.bo_is_reusable = mos_gem_bo_is_reusable;
3873 bufmgr_gem->bufmgr.bo_references = mos_gem_bo_references;
3874
3875 bufmgr_gem->bufmgr.bo_wait = mos_gem_bo_wait;
3876 bufmgr_gem->bufmgr.bo_clear_relocs = mos_gem_bo_clear_relocs;
3877 bufmgr_gem->bufmgr.context_create = mos_gem_context_create;
3878 bufmgr_gem->bufmgr.context_create_ext = mos_gem_context_create_ext;
3879 bufmgr_gem->bufmgr.context_create_shared = mos_gem_context_create_shared;
3880 bufmgr_gem->bufmgr.context_destroy = mos_gem_context_destroy;
3881 bufmgr_gem->bufmgr.vm_create = mos_gem_vm_create;
3882 bufmgr_gem->bufmgr.vm_destroy = mos_gem_vm_destroy;
3883 bufmgr_gem->bufmgr.bo_context_exec2 = mos_gem_bo_context_exec2;
3884 bufmgr_gem->bufmgr.bo_is_softpin = mos_gem_bo_is_softpin;
3885 bufmgr_gem->bufmgr.bo_map_gtt = mos_gem_bo_map_gtt;
3886 bufmgr_gem->bufmgr.bo_unmap_gtt = mos_gem_bo_unmap_gtt;
3887 bufmgr_gem->bufmgr.bo_map_wc = mos_gem_bo_map_wc;
3888 bufmgr_gem->bufmgr.bo_unmap_gtt = mos_gem_bo_unmap_wc;
3889 bufmgr_gem->bufmgr.bo_map_unsynchronized = mos_gem_bo_map_unsynchronized;
3890 bufmgr_gem->bufmgr.bo_start_gtt_access = mos_gem_bo_start_gtt_access;
3891 bufmgr_gem->bufmgr.enable_reuse = mos_gem_enable_reuse;
3892 bufmgr_gem->bufmgr.get_devid = mos_gem_get_devid;
3893 bufmgr_gem->bufmgr.set_context_param = mos_gem_set_context_param;
3894 bufmgr_gem->bufmgr.set_context_param_load_balance = mos_gem_set_context_param_load_balance;
3895 bufmgr_gem->bufmgr.get_context_param = mos_gem_get_context_param;
3896 bufmgr_gem->bufmgr.bo_create_from_prime = mos_gem_bo_create_from_prime;
3897 bufmgr_gem->bufmgr.bo_export_to_prime = mos_gem_bo_export_to_prime;
3898 bufmgr_gem->bufmgr.reg_read = mos_bufmg_reg_read;
3899 bufmgr_gem->bufmgr.get_reset_stats = mos_bufmg_get_reset_stats;
3900 bufmgr_gem->bufmgr.get_context_param_sseu = mos_bufmgr_get_context_param_sseu;
3901 bufmgr_gem->bufmgr.set_context_param_sseu = mos_bufmgr_set_context_param_sseu;
3902 bufmgr_gem->bufmgr.get_platform_information = mos_bufmgr_get_platform_information;
3903 bufmgr_gem->bufmgr.set_platform_information = mos_bufmgr_set_platform_information;
3904 bufmgr_gem->bufmgr.switch_off_n_bits = mos_bufmgr_switch_off_n_bits;
3905 bufmgr_gem->bufmgr.hweight8 = mos_bufmgr_hweight8;
3906 bufmgr_gem->bufmgr.query_engines = mos_bufmgr_query_engines;
3907
3908 memclear(aperture);
3909 ret = drmIoctl(bufmgr_gem->fd,
3910 DRM_IOCTL_I915_GEM_GET_APERTURE,
3911 &aperture);
3912
3913 if (ret == 0)
3914 bufmgr_gem->gtt_size = aperture.aper_available_size;
3915 else {
3916 fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
3917 strerror(errno));
3918 bufmgr_gem->gtt_size = 128 * 1024 * 1024;
3919 fprintf(stderr, "Assuming %dkB available aperture size.\n"
3920 "May lead to reduced performance or incorrect "
3921 "rendering.\n",
3922 (int)bufmgr_gem->gtt_size / 1024);
3923 }
3924
3925 bufmgr_gem->pci_device = get_pci_device_id(bufmgr_gem);
3926
3927 memclear(gp);
3928 gp.value = &tmp;
3929
3930 gp.param = I915_PARAM_HAS_EXECBUF2;
3931 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3932 if (!ret)
3933 exec2 = true;
3934
3935 gp.param = I915_PARAM_HAS_BSD;
3936 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3937 bufmgr_gem->has_bsd = ret == 0;
3938
3939 gp.param = I915_PARAM_HAS_BLT;
3940 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3941 bufmgr_gem->has_blt = ret == 0;
3942
3943 gp.param = I915_PARAM_HAS_RELAXED_FENCING;
3944 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3945 bufmgr_gem->has_relaxed_fencing = ret == 0;
3946
3947 bufmgr_gem->bufmgr.bo_alloc_userptr = check_bo_alloc_userptr;
3948
3949 gp.param = I915_PARAM_HAS_WAIT_TIMEOUT;
3950 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3951 bufmgr_gem->has_wait_timeout = ret == 0;
3952
3953 gp.param = I915_PARAM_HAS_LLC;
3954 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3955 bufmgr_gem->has_llc = *gp.value;
3956
3957 gp.param = I915_PARAM_HAS_VEBOX;
3958 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3959 bufmgr_gem->has_vebox = (ret == 0) & (*gp.value > 0);
3960
3961 gp.param = I915_PARAM_MMAP_VERSION;
3962 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3963 bufmgr_gem->has_ext_mmap = (ret == 0) & (*gp.value > 0);
3964
3965 gp.param = I915_PARAM_HAS_EXEC_SOFTPIN;
3966 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3967 if (ret == 0 && *gp.value > 0)
3968 {
3969 bufmgr_gem->bufmgr.bo_set_softpin = mos_gem_bo_set_softpin;
3970 bufmgr_gem->bufmgr.bo_add_softpin_target = mos_gem_bo_add_softpin_target;
3971 }
3972
3973 gp.param = I915_PARAM_HAS_EXEC_ASYNC;
3974 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3975 if (ret == 0 && *gp.value > 0){
3976 bufmgr_gem->bufmgr.set_object_async = mos_gem_bo_set_object_async;
3977 bufmgr_gem->bufmgr.set_exec_object_async = mos_gem_bo_set_exec_object_async;
3978 }
3979
3980 gp.param = I915_PARAM_HAS_ALIASING_PPGTT;
3981 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
3982 if (ret == 0 && *gp.value == 3)
3983 bufmgr_gem->bufmgr.bo_use_48b_address_range = mos_gem_bo_use_48b_address_range;
3984
3985 /* Let's go with one relocation per every 2 dwords (but round down a bit
3986 * since a power of two will mean an extra page allocation for the reloc
3987 * buffer).
3988 *
3989 * Every 4 was too few for the blender benchmark.
3990 */
3991 bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
3992 bufmgr_gem->device_type = DEVICE_TYPE_I915;
3993
3994 DRMINITLISTHEAD(&bufmgr_gem->named);
3995 init_cache_buckets(bufmgr_gem);
3996
3997 DRMLISTADD(&bufmgr_gem->managers, &bufmgr_list);
3998 bufmgr_gem->use_softpin = false;
3999 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_SYS], MEMZONE_SYS_START, MEMZONE_SYS_SIZE);
4000 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_DEVICE], MEMZONE_DEVICE_START, MEMZONE_DEVICE_SIZE);
4001 exit:
4002 pthread_mutex_unlock(&bufmgr_list_mutex);
4003
4004 return bufmgr_gem != nullptr ? &bufmgr_gem->bufmgr : nullptr;
4005 }
4006
4007