1 /**************************************************************************
2 *
3 * Copyright © 2007 Red Hat Inc.
4 * Copyright © 2007-2022 Intel Corporation
5 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * The above copyright notice and this permission notice (including the
25 * next paragraph) shall be included in all copies or substantial portions
26 * of the Software.
27 *
28 *
29 **************************************************************************/
30 /*
31 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
32 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
33 * Eric Anholt <[email protected]>
34 * Dave Airlie <[email protected]>
35 */
36
37 #ifdef HAVE_CONFIG_H
38 #include "config.h"
39 #endif
40
41 #include "xf86drm.h"
42 #include "xf86atomic.h"
43 #include <fcntl.h>
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <string.h>
47 #include <unistd.h>
48 #include <assert.h>
49 #include <pthread.h>
50 #include <sys/ioctl.h>
51 #include <sys/stat.h>
52 #include <sys/types.h>
53 #include <stdbool.h>
54
55 #include "errno.h"
56 #ifndef ETIME
57 #define ETIME ETIMEDOUT
58 #endif
59 #include "libdrm_macros.h"
60 #include "libdrm_lists.h"
61 #include "mos_bufmgr.h"
62 #include "mos_bufmgr_priv.h"
63 #ifdef ENABLE_XE_KMD
64 #include "mos_bufmgr_xe.h"
65 #endif
66 #include "string.h"
67
68 #include "i915_drm.h"
69 #include "mos_vma.h"
70 #include "mos_util_debug.h"
71 #include "mos_oca_defs_specific.h"
72 #include "intel_hwconfig_types.h"
73 #include "mos_utilities.h"
74 #include "linux_system_info.h"
75 #include "mos_os_specific.h"
76
77 #ifdef HAVE_VALGRIND
78 #include <valgrind.h>
79 #include <memcheck.h>
80 #define VG(x) x
81 #else
82 #define VG(x)
83 #endif
84
85 #define memclear(s) memset(&s, 0, sizeof(s))
86
87 #define MOS_DBG(...) do { \
88 if (bufmgr_gem != nullptr && bufmgr_gem->bufmgr.debug) \
89 fprintf(stderr, __VA_ARGS__); \
90 } while (0)
91
92 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
93 #define MAX2(A, B) ((A) > (B) ? (A) : (B))
94
95 /**
96 * upper_32_bits - return bits 32-63 of a number
97 * @n: the number we're accessing
98 *
99 * A basic shift-right of a 64- or 32-bit quantity. Use this to suppress
100 * the "right shift count >= width of type" warning when that quantity is
101 * 32-bits.
102 */
103 #define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))
104
105 /**
106 * lower_32_bits - return bits 0-31 of a number
107 * @n: the number we're accessing
108 */
109 #define lower_32_bits(n) ((__u32)(n))
110
111 #define PCI_CHIP_I915_G 0x2582
112 #define PCI_CHIP_E7221_G 0x258A
113 #define PCI_CHIP_I915_GM 0x2592
114
115 #define IS_915(devid) ((devid) == PCI_CHIP_I915_G || \
116 (devid) == PCI_CHIP_E7221_G || \
117 (devid) == PCI_CHIP_I915_GM)
118
119 #define INITIAL_SOFTPIN_TARGET_COUNT 1024
120
121 struct mos_gem_bo_bucket {
122 drmMMListHead head;
123 unsigned long size;
124 };
125
126 struct mos_bufmgr_gem {
127 struct mos_bufmgr bufmgr;
128
129 atomic_t refcount;
130
131 int fd;
132
133 int max_relocs;
134
135 pthread_mutex_t lock;
136
137 struct drm_i915_gem_exec_object *exec_objects;
138 struct drm_i915_gem_exec_object2 *exec2_objects;
139 struct mos_linux_bo **exec_bos;
140 int exec_size;
141 int exec_count;
142
143 /** Array of lists of cached gem objects of power-of-two sizes */
144 struct mos_gem_bo_bucket cache_bucket[64];
145 int num_buckets;
146 time_t time;
147
148 drmMMListHead managers;
149
150 drmMMListHead named;
151
152 uint64_t gtt_size;
153 int available_fences;
154 int pci_device;
155 unsigned int has_bsd : 1;
156 unsigned int has_bsd2 : 1;
157 unsigned int has_blt : 1;
158 unsigned int has_relaxed_fencing : 1;
159 unsigned int has_llc : 1;
160 unsigned int has_wait_timeout : 1;
161 unsigned int bo_reuse : 1;
162 unsigned int no_exec : 1;
163 unsigned int has_vebox : 1;
164 unsigned int has_ext_mmap : 1;
165 unsigned int has_fence_reg : 1;
166 unsigned int has_lmem : 1;
167 unsigned int has_mmap_offset : 1;
168 bool fenced_relocs;
169
170 struct {
171 void *ptr;
172 uint32_t handle;
173 } userptr_active;
174
175 // manage address for softpin buffer object
176 mos_vma_heap vma_heap[MEMZONE_COUNT];
177 bool use_softpin;
178 bool softpin_va1Malign;
179
180 bool object_capture_disabled;
181
182 #define MEM_PROFILER_BUFFER_SIZE 256
183 char mem_profiler_buffer[MEM_PROFILER_BUFFER_SIZE];
184 char* mem_profiler_path;
185 int mem_profiler_fd;
186
187 int device_type;
188
189 uint32_t ts_freq;
190 } mos_bufmgr_gem;
191
192 #define DRM_INTEL_RELOC_FENCE (1<<0)
193
194 struct mos_reloc_target {
195 struct mos_linux_bo *bo;
196 int flags;
197 };
198
199 struct mos_softpin_target {
200 struct mos_linux_bo *bo;
201 int flags;
202 };
203
204 struct mos_bo_gem {
205 struct mos_linux_bo bo;
206
207 atomic_t refcount;
208 uint32_t gem_handle;
209 const char *name;
210
211 /**
212 * Kenel-assigned global name for this object
213 *
214 * List contains both flink named and prime fd'd objects
215 */
216 unsigned int global_name;
217 drmMMListHead name_list;
218
219 /**
220 * Index of the buffer within the validation list while preparing a
221 * batchbuffer execution.
222 */
223 int validate_index;
224
225 /**
226 * Current tiling mode
227 */
228 uint32_t tiling_mode;
229 uint32_t swizzle_mode;
230 unsigned long stride;
231
232 time_t free_time;
233
234 /** Array passed to the DRM containing relocation information. */
235 struct drm_i915_gem_relocation_entry *relocs;
236 /**
237 * Array of info structs corresponding to relocs[i].target_handle etc
238 */
239 struct mos_reloc_target *reloc_target_info;
240 /** Number of entries in relocs */
241 int reloc_count;
242 /** Array of BOs that are referenced by this buffer and will be softpinned */
243 struct mos_softpin_target *softpin_target;
244 /** Number softpinned BOs that are referenced by this buffer */
245 int softpin_target_count;
246 /** Maximum amount of softpinned BOs that are referenced by this buffer */
247 int max_softpin_target_count;
248
249 /** Mapped address for the buffer, saved across map/unmap cycles */
250 void *mem_virtual;
251 /** Uncached Mapped address for the buffer, saved across map/unmap cycles */
252 void *mem_wc_virtual;
253 /** GTT virtual address for the buffer, saved across map/unmap cycles */
254 void *gtt_virtual;
255 /**
256 * Virtual address of the buffer allocated by user, used for userptr
257 * objects only.
258 */
259 void *user_virtual;
260 int map_count;
261
262 /** BO cache list */
263 drmMMListHead head;
264
265 /**
266 * Boolean of whether this BO and its children have been included in
267 * the current drm_intel_bufmgr_check_aperture_space() total.
268 */
269 bool included_in_check_aperture;
270
271 /**
272 * Boolean of whether this buffer has been used as a relocation
273 * target and had its size accounted for, and thus can't have any
274 * further relocations added to it.
275 */
276 bool used_as_reloc_target;
277
278 /**
279 * Boolean of whether we have encountered an error whilst building the relocation tree.
280 */
281 bool has_error;
282
283 /**
284 * Boolean of whether this buffer can be re-used
285 */
286 bool reusable;
287
288 /**
289 * Boolean of whether the GPU is definitely not accessing the buffer.
290 *
291 * This is only valid when reusable, since non-reusable
292 * buffers are those that have been shared wth other
293 * processes, so we don't know their state.
294 */
295 bool idle;
296
297 /**
298 * Boolean of whether this buffer was allocated with userptr
299 */
300 bool is_userptr;
301
302 /**
303 * Boolean of whether this buffer can be placed in the full 48-bit
304 * address range on gen8+.
305 *
306 * By default, buffers will be keep in a 32-bit range, unless this
307 * flag is explicitly set.
308 */
309 bool use_48b_address_range;
310
311 /**
312 * Whether this buffer is softpinned at offset specified by the user
313 */
314 bool is_softpin;
315
316 /*
317 * Whether to remove the dependency of this bo in exebuf.
318 */
319 bool exec_async;
320
321 /*
322 * Whether to remove the dependency of this bo in exebuf.
323 */
324 bool exec_capture;
325
326 /**
327 * Size in bytes of this buffer and its relocation descendents.
328 *
329 * Used to avoid costly tree walking in
330 * drm_intel_bufmgr_check_aperture in the common case.
331 */
332 int reloc_tree_size;
333
334 /**
335 * Number of potential fence registers required by this buffer and its
336 * relocations.
337 */
338 int reloc_tree_fences;
339
340 /** Flags that we may need to do the SW_FINSIH ioctl on unmap. */
341 bool mapped_cpu_write;
342
343 /**
344 * Size to pad the object to.
345 *
346 */
347 uint64_t pad_to_size;
348
349 /**
350 * Memory Type on created the surfaces for local/system memory
351 */
352 int mem_region;
353
354 /**
355 * PAT Index
356 */
357 unsigned int pat_index;
358
359 /**
360 * Is cpu cacheable
361 */
362 bool cpu_cacheable;
363 };
364
365 struct mos_exec_info {
366 /* save all exec2_objects for res*/
367 struct drm_i915_gem_exec_object2* obj;
368 /* save batch buffer*/
369 struct drm_i915_gem_exec_object2* batch_obj;
370 /* save previous ptr to void mem leak when free*/
371 struct drm_i915_gem_exec_object2* pSavePreviousExec2Objects;
372 /*bo resource count*/
373 uint32_t obj_count;
374 /*batch buffer bo count*/
375 uint32_t batch_count;
376 /*remain size of 'obj'*/
377 uint32_t obj_remain_size;
378 #define OBJ512_SIZE 512
379 };
380
381 static unsigned int
382 mos_gem_estimate_batch_space(struct mos_linux_bo ** bo_array, int count);
383
384 static unsigned int
385 mos_gem_compute_batch_space(struct mos_linux_bo ** bo_array, int count);
386
387 static int
388 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
389 uint32_t * swizzle_mode);
390
391 static int
392 mos_gem_bo_check_mem_region_internal(struct mos_linux_bo *bo,
393 int mem_type);
394
395 static int
396 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
397 uint32_t tiling_mode,
398 uint32_t stride);
399
400 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
401 time_t time);
402
403 static void mos_gem_bo_unreference(struct mos_linux_bo *bo);
404 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo);
405 static void mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable);
406 static void mos_gem_bo_free(struct mos_linux_bo *bo);
407
408 static int mos_bufmgr_get_driver_info(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo);
409
to_bo_gem(struct mos_linux_bo * bo)410 static inline struct mos_bo_gem *to_bo_gem(struct mos_linux_bo *bo)
411 {
412 return (struct mos_bo_gem *)bo;
413 }
414
415 static unsigned long
mos_gem_bo_tile_size(struct mos_bufmgr_gem * bufmgr_gem,unsigned long size,uint32_t * tiling_mode)416 mos_gem_bo_tile_size(struct mos_bufmgr_gem *bufmgr_gem, unsigned long size,
417 uint32_t *tiling_mode)
418 {
419 unsigned long min_size, max_size;
420 unsigned long i;
421
422 if (*tiling_mode == I915_TILING_NONE)
423 return size;
424
425 /* 965+ just need multiples of page size for tiling */
426 return ROUND_UP_TO(size, 4096);
427
428 }
429
430 /*
431 * Round a given pitch up to the minimum required for X tiling on a
432 * given chip. We use 512 as the minimum to allow for a later tiling
433 * change.
434 */
435 static unsigned long
mos_gem_bo_tile_pitch(struct mos_bufmgr_gem * bufmgr_gem,unsigned long pitch,uint32_t * tiling_mode)436 mos_gem_bo_tile_pitch(struct mos_bufmgr_gem *bufmgr_gem,
437 unsigned long pitch, uint32_t *tiling_mode)
438 {
439 unsigned long tile_width;
440 unsigned long i;
441
442 /* If untiled, then just align it so that we can do rendering
443 * to it with the 3D engine.
444 */
445 if (*tiling_mode == I915_TILING_NONE)
446 return ALIGN(pitch, 64);
447
448 if (*tiling_mode == I915_TILING_X
449 || (IS_915(bufmgr_gem->pci_device)
450 && *tiling_mode == I915_TILING_Y))
451 tile_width = 512;
452 else
453 tile_width = 128;
454
455 /* 965 is flexible */
456 return ROUND_UP_TO(pitch, tile_width);
457 }
458
459 static struct mos_gem_bo_bucket *
mos_gem_bo_bucket_for_size(struct mos_bufmgr_gem * bufmgr_gem,unsigned long size)460 mos_gem_bo_bucket_for_size(struct mos_bufmgr_gem *bufmgr_gem,
461 unsigned long size)
462 {
463 int i;
464
465 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
466 struct mos_gem_bo_bucket *bucket =
467 &bufmgr_gem->cache_bucket[i];
468 if (bucket->size >= size) {
469 return bucket;
470 }
471 }
472
473 return nullptr;
474 }
475
476 static void
mos_gem_dump_validation_list(struct mos_bufmgr_gem * bufmgr_gem)477 mos_gem_dump_validation_list(struct mos_bufmgr_gem *bufmgr_gem)
478 {
479 int i, j;
480
481 for (i = 0; i < bufmgr_gem->exec_count; i++) {
482 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
483 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
484
485 if (bo_gem->relocs == nullptr || bo_gem->softpin_target == nullptr) {
486 MOS_DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,
487 bo_gem->is_softpin ? "*" : "",
488 bo_gem->name);
489 continue;
490 }
491
492 for (j = 0; j < bo_gem->reloc_count; j++) {
493 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[j].bo;
494 struct mos_bo_gem *target_gem =
495 (struct mos_bo_gem *) target_bo;
496
497 MOS_DBG("%2d: %d %s(%s)@0x%08x %08x -> "
498 "%d (%s)@0x%08x %08x + 0x%08x\n",
499 i,
500 bo_gem->gem_handle,
501 bo_gem->is_softpin ? "*" : "",
502 bo_gem->name,
503 upper_32_bits(bo_gem->relocs[j].offset),
504 lower_32_bits(bo_gem->relocs[j].offset),
505 target_gem->gem_handle,
506 target_gem->name,
507 upper_32_bits(target_bo->offset64),
508 lower_32_bits(target_bo->offset64),
509 bo_gem->relocs[j].delta);
510 }
511
512 for (j = 0; j < bo_gem->softpin_target_count; j++) {
513 struct mos_linux_bo *target_bo = bo_gem->softpin_target[j].bo;
514 struct mos_bo_gem *target_gem =
515 (struct mos_bo_gem *) target_bo;
516 MOS_DBG("%2d: %d %s(%s) -> "
517 "%d *(%s)@0x%08x %08x\n",
518 i,
519 bo_gem->gem_handle,
520 bo_gem->is_softpin ? "*" : "",
521 bo_gem->name,
522 target_gem->gem_handle,
523 target_gem->name,
524 upper_32_bits(target_bo->offset64),
525 lower_32_bits(target_bo->offset64));
526 }
527 }
528 }
529
530 static inline void
mos_gem_bo_reference(struct mos_linux_bo * bo)531 mos_gem_bo_reference(struct mos_linux_bo *bo)
532 {
533 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
534
535 atomic_inc(&bo_gem->refcount);
536 }
537
538 /**
539 * Adds the given buffer to the list of buffers to be validated (moved into the
540 * appropriate memory type) with the next batch submission.
541 *
542 * If a buffer is validated multiple times in a batch submission, it ends up
543 * with the intersection of the memory type flags and the union of the
544 * access flags.
545 */
546 static void
mos_add_validate_buffer(struct mos_linux_bo * bo)547 mos_add_validate_buffer(struct mos_linux_bo *bo)
548 {
549 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
550 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
551 int index;
552 struct drm_i915_gem_exec_object *exec_objects;
553 struct mos_linux_bo **exec_bos;
554
555 if (bo_gem->validate_index != -1)
556 return;
557
558 /* Extend the array of validation entries as necessary. */
559 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
560 int new_size = bufmgr_gem->exec_size * 2;
561
562 if (new_size == 0)
563 new_size = ARRAY_INIT_SIZE;
564
565 exec_objects = (struct drm_i915_gem_exec_object *)realloc(bufmgr_gem->exec_objects,
566 sizeof(*bufmgr_gem->exec_objects) * new_size);
567 if (!exec_objects)
568 return;
569
570 bufmgr_gem->exec_objects = exec_objects;
571
572 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
573 sizeof(*bufmgr_gem->exec_bos) * new_size);
574 if (!exec_bos)
575 return;
576
577 bufmgr_gem->exec_bos = exec_bos;
578 bufmgr_gem->exec_size = new_size;
579 }
580
581 index = bufmgr_gem->exec_count;
582 bo_gem->validate_index = index;
583 /* Fill in array entry */
584 bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;
585 bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;
586 bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;
587 bufmgr_gem->exec_objects[index].alignment = bo->align;
588 bufmgr_gem->exec_objects[index].offset = 0;
589 bufmgr_gem->exec_bos[index] = bo;
590 bufmgr_gem->exec_count++;
591 }
592
593 static void
mos_add_validate_buffer2(struct mos_linux_bo * bo,int need_fence)594 mos_add_validate_buffer2(struct mos_linux_bo *bo, int need_fence)
595 {
596 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
597 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
598 int index;
599 struct drm_i915_gem_exec_object2 *exec2_objects;
600 struct mos_linux_bo **exec_bos;
601 int flags = 0;
602
603 if (need_fence)
604 flags |= EXEC_OBJECT_NEEDS_FENCE;
605 if (bo_gem->pad_to_size)
606 flags |= EXEC_OBJECT_PAD_TO_SIZE;
607 if (bo_gem->use_48b_address_range)
608 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
609 if (bo_gem->is_softpin)
610 flags |= EXEC_OBJECT_PINNED;
611 if (bo_gem->exec_async)
612 flags |= EXEC_OBJECT_ASYNC;
613 if (bo_gem->exec_capture)
614 flags |= EXEC_OBJECT_CAPTURE;
615
616 if (bo_gem->validate_index != -1) {
617 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= flags;
618 return;
619 }
620
621 /* Extend the array of validation entries as necessary. */
622 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
623 int new_size = bufmgr_gem->exec_size * 2;
624
625 if (new_size == 0)
626 new_size = ARRAY_INIT_SIZE;
627 exec2_objects = (struct drm_i915_gem_exec_object2 *)
628 realloc(bufmgr_gem->exec2_objects,
629 sizeof(*bufmgr_gem->exec2_objects) * new_size);
630 if (!exec2_objects)
631 return;
632
633 bufmgr_gem->exec2_objects = exec2_objects;
634
635 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
636 sizeof(*bufmgr_gem->exec_bos) * new_size);
637 if (!exec_bos)
638 return;
639
640 bufmgr_gem->exec_bos = exec_bos;
641 bufmgr_gem->exec_size = new_size;
642 }
643
644 index = bufmgr_gem->exec_count;
645 bo_gem->validate_index = index;
646 /* Fill in array entry */
647 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
648 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
649 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
650 bufmgr_gem->exec2_objects[index].alignment = bo->align;
651 bufmgr_gem->exec2_objects[index].offset = bo_gem->is_softpin ?
652 bo->offset64 : 0;
653 bufmgr_gem->exec_bos[index] = bo;
654 bufmgr_gem->exec2_objects[index].flags = flags;
655 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
656 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
657 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
658 bufmgr_gem->exec_count++;
659 }
660
661 static void
mos_add_reloc_objects(struct mos_reloc_target reloc_target)662 mos_add_reloc_objects(struct mos_reloc_target reloc_target)
663 {
664 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)reloc_target.bo->bufmgr;
665 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)reloc_target.bo;
666 int index;
667 struct drm_i915_gem_exec_object2 *exec2_objects;
668 struct mos_linux_bo **exec_bos;
669
670 if (bo_gem->validate_index != -1) {
671 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= reloc_target.flags;
672 return;
673 }
674
675 /* Extend the array of validation entries as necessary. */
676 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
677 int new_size = bufmgr_gem->exec_size * 2;
678
679 if (new_size == 0)
680 new_size = ARRAY_INIT_SIZE;
681 exec2_objects = (struct drm_i915_gem_exec_object2 *)
682 realloc(bufmgr_gem->exec2_objects,
683 sizeof(*bufmgr_gem->exec2_objects) * new_size);
684 if (!exec2_objects)
685 {
686 MOS_DBG("realloc exec2_objects failed!\n");
687 return;
688 }
689
690 bufmgr_gem->exec2_objects = exec2_objects;
691
692 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
693 sizeof(*bufmgr_gem->exec_bos) * new_size);
694 if (!exec_bos)
695 {
696 MOS_DBG("realloc exec_bo failed!\n");
697 return;
698 }
699
700 bufmgr_gem->exec_bos = exec_bos;
701 bufmgr_gem->exec_size = new_size;
702 }
703
704 index = bufmgr_gem->exec_count;
705 bo_gem->validate_index = index;
706 /* Fill in array entry */
707 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
708 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
709 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
710 bufmgr_gem->exec2_objects[index].alignment = reloc_target.bo->align;
711 bufmgr_gem->exec2_objects[index].offset = 0;
712 bufmgr_gem->exec_bos[index] = reloc_target.bo;
713 bufmgr_gem->exec2_objects[index].flags = reloc_target.flags;
714 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
715 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
716 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
717 bufmgr_gem->exec_count++;
718 }
719
720 static void
mos_add_softpin_objects(struct mos_softpin_target softpin_target)721 mos_add_softpin_objects(struct mos_softpin_target softpin_target)
722 {
723 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)softpin_target.bo->bufmgr;
724 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)softpin_target.bo;
725 int index;
726 struct drm_i915_gem_exec_object2 *exec2_objects;
727 struct mos_linux_bo **exec_bos;
728
729 if (bo_gem->validate_index != -1) {
730 bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= softpin_target.flags;
731 return;
732 }
733
734 /* Extend the array of validation entries as necessary. */
735 if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {
736 int new_size = bufmgr_gem->exec_size * 2;
737
738 if (new_size == 0)
739 new_size = ARRAY_INIT_SIZE;
740 exec2_objects = (struct drm_i915_gem_exec_object2 *)
741 realloc(bufmgr_gem->exec2_objects,
742 sizeof(*bufmgr_gem->exec2_objects) * new_size);
743 if (!exec2_objects)
744 {
745 MOS_DBG("realloc exec2_objects failed!\n");
746 return;
747 }
748
749 bufmgr_gem->exec2_objects = exec2_objects;
750
751 exec_bos = (struct mos_linux_bo **)realloc(bufmgr_gem->exec_bos,
752 sizeof(*bufmgr_gem->exec_bos) * new_size);
753 if (!exec_bos)
754 {
755 MOS_DBG("realloc exec_bo failed!\n");
756 return;
757 }
758
759 bufmgr_gem->exec_bos = exec_bos;
760 bufmgr_gem->exec_size = new_size;
761 }
762
763 index = bufmgr_gem->exec_count;
764 bo_gem->validate_index = index;
765 /* Fill in array entry */
766 bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;
767 bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;
768 bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;
769 bufmgr_gem->exec2_objects[index].alignment = softpin_target.bo->align;
770 bufmgr_gem->exec2_objects[index].offset = softpin_target.bo->offset64;
771 bufmgr_gem->exec2_objects[index].flags = softpin_target.flags;
772 bufmgr_gem->exec2_objects[index].pad_to_size = bo_gem->pad_to_size;
773 bufmgr_gem->exec2_objects[index].rsvd1 = 0;
774 bufmgr_gem->exec2_objects[index].rsvd2 = 0;
775 bufmgr_gem->exec_bos[index] = softpin_target.bo;
776 bufmgr_gem->exec_count++;
777 }
778
779 #define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \
780 sizeof(uint32_t))
781
782 static void
mos_bo_gem_set_in_aperture_size(struct mos_bufmgr_gem * bufmgr_gem,struct mos_bo_gem * bo_gem,unsigned int alignment)783 mos_bo_gem_set_in_aperture_size(struct mos_bufmgr_gem *bufmgr_gem,
784 struct mos_bo_gem *bo_gem,
785 unsigned int alignment)
786 {
787 unsigned int size;
788
789 assert(!bo_gem->used_as_reloc_target);
790
791 /* The older chipsets are far-less flexible in terms of tiling,
792 * and require tiled buffer to be size aligned in the aperture.
793 * This means that in the worst possible case we will need a hole
794 * twice as large as the object in order for it to fit into the
795 * aperture. Optimal packing is for wimps.
796 */
797 size = bo_gem->bo.size;
798
799 bo_gem->reloc_tree_size = size + alignment;
800 }
801
802 static int
mos_setup_reloc_list(struct mos_linux_bo * bo)803 mos_setup_reloc_list(struct mos_linux_bo *bo)
804 {
805 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
806 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
807 unsigned int max_relocs = bufmgr_gem->max_relocs;
808
809 if (bo->size / 4 < max_relocs)
810 max_relocs = bo->size / 4;
811
812 bo_gem->relocs = (struct drm_i915_gem_relocation_entry *)malloc(max_relocs *
813 sizeof(struct drm_i915_gem_relocation_entry));
814 bo_gem->reloc_target_info = (struct mos_reloc_target *)malloc(max_relocs *
815 sizeof(struct mos_reloc_target));
816 if (bo_gem->relocs == nullptr || bo_gem->reloc_target_info == nullptr) {
817 bo_gem->has_error = true;
818
819 free (bo_gem->relocs);
820 bo_gem->relocs = nullptr;
821
822 free (bo_gem->reloc_target_info);
823 bo_gem->reloc_target_info = nullptr;
824
825 return 1;
826 }
827
828 return 0;
829 }
830
831 static int
mos_gem_bo_busy(struct mos_linux_bo * bo)832 mos_gem_bo_busy(struct mos_linux_bo *bo)
833 {
834 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
835 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
836 struct drm_i915_gem_busy busy;
837 int ret;
838
839 if (bo_gem->reusable && bo_gem->idle)
840 return false;
841
842 memclear(busy);
843 busy.handle = bo_gem->gem_handle;
844
845 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
846 if (ret == 0) {
847 bo_gem->idle = !busy.busy;
848 return busy.busy;
849 } else {
850 return false;
851 }
852 return (ret == 0 && busy.busy);
853 }
854
855 static int
mos_gem_bo_madvise_internal(struct mos_bufmgr_gem * bufmgr_gem,struct mos_bo_gem * bo_gem,int state)856 mos_gem_bo_madvise_internal(struct mos_bufmgr_gem *bufmgr_gem,
857 struct mos_bo_gem *bo_gem, int state)
858 {
859 struct drm_i915_gem_madvise madv;
860
861 memclear(madv);
862 madv.handle = bo_gem->gem_handle;
863 madv.madv = state;
864 madv.retained = 1;
865 drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
866
867 return madv.retained;
868 }
869
870 static int
mos_gem_bo_madvise(struct mos_linux_bo * bo,int madv)871 mos_gem_bo_madvise(struct mos_linux_bo *bo, int madv)
872 {
873 return mos_gem_bo_madvise_internal
874 ((struct mos_bufmgr_gem *) bo->bufmgr,
875 (struct mos_bo_gem *) bo,
876 madv);
877 }
878
879 /* drop the oldest entries that have been purged by the kernel */
880 static void
mos_gem_bo_cache_purge_bucket(struct mos_bufmgr_gem * bufmgr_gem,struct mos_gem_bo_bucket * bucket)881 mos_gem_bo_cache_purge_bucket(struct mos_bufmgr_gem *bufmgr_gem,
882 struct mos_gem_bo_bucket *bucket)
883 {
884 while (!DRMLISTEMPTY(&bucket->head)) {
885 struct mos_bo_gem *bo_gem;
886
887 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
888 bucket->head.next, head);
889 if (mos_gem_bo_madvise_internal
890 (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))
891 break;
892
893 DRMLISTDEL(&bo_gem->head);
894 mos_gem_bo_free(&bo_gem->bo);
895 }
896 }
897
898 static int
mos_gem_query_items(int fd,struct drm_i915_query_item * items,uint32_t n_items)899 mos_gem_query_items(int fd, struct drm_i915_query_item *items, uint32_t n_items)
900 {
901 struct drm_i915_query q;
902
903 memclear(q);
904 q.num_items = n_items;
905 q.items_ptr = (uintptr_t)items;
906
907 return drmIoctl(fd, DRM_IOCTL_I915_QUERY, &q);
908 }
909
910 /**
911 * query mechanism for memory regions.
912 */
mos_gem_get_query_memory_regions(int fd)913 static struct drm_i915_query_memory_regions *mos_gem_get_query_memory_regions(int fd)
914 {
915 int ret;
916 struct drm_i915_query_item item;
917 struct drm_i915_query_memory_regions *query_info;
918
919 memclear(item);
920 item.query_id = DRM_I915_QUERY_MEMORY_REGIONS;
921 ret = mos_gem_query_items(fd, &item, 1);
922 if (ret != 0 || item.length <= 0)
923 return NULL;
924
925 query_info = (drm_i915_query_memory_regions*)calloc(1, item.length);
926
927 item.data_ptr = (uintptr_t)query_info;
928 ret = mos_gem_query_items(fd, &item, 1);
929 if (ret != 0) {
930 free(query_info);
931 return NULL;
932 }
933
934 return query_info;
935 }
936
937 /**
938 * check how many lmem regions are available on device.
939 */
mos_gem_get_lmem_region_count(int fd)940 static uint8_t mos_gem_get_lmem_region_count(int fd)
941 {
942 struct drm_i915_query_memory_regions *query_info;
943 uint8_t num_regions = 0;
944 uint8_t lmem_regions = 0;
945
946 query_info = mos_gem_get_query_memory_regions(fd);
947
948 if(query_info)
949 {
950 num_regions = query_info->num_regions;
951
952 for (int i = 0; i < num_regions; i++) {
953 if (query_info->regions[i].region.memory_class == I915_MEMORY_CLASS_DEVICE)
954 {
955 lmem_regions += 1;
956 }
957 }
958 free(query_info);
959 }
960
961 return lmem_regions;
962 }
963
964 /**
965 * check if lmem is available on device.
966 */
mos_gem_has_lmem(int fd)967 static bool mos_gem_has_lmem(int fd)
968 {
969 return mos_gem_get_lmem_region_count(fd) > 0;
970 }
971
972 static enum mos_memory_zone
mos_gem_bo_memzone_for_address(uint64_t address)973 mos_gem_bo_memzone_for_address(uint64_t address)
974 {
975 if (address >= MEMZONE_DEVICE_START)
976 return MEMZONE_DEVICE;
977 else
978 return MEMZONE_SYS;
979 }
980
981 /**
982 * Allocate a section of virtual memory for a buffer, assigning an address.
983 */
984 static uint64_t
mos_gem_bo_vma_alloc(struct mos_bufmgr * bufmgr,enum mos_memory_zone memzone,uint64_t size,uint64_t alignment)985 mos_gem_bo_vma_alloc(struct mos_bufmgr *bufmgr,
986 enum mos_memory_zone memzone,
987 uint64_t size,
988 uint64_t alignment)
989 {
990 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", 0);
991 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
992 /* Force alignment to be some number of pages */
993 alignment = ALIGN(alignment, PAGE_SIZE);
994
995 uint64_t addr = mos_vma_heap_alloc(&bufmgr_gem->vma_heap[memzone], size, alignment);
996
997 // currently only support 48bit range address
998 CHK_CONDITION((addr >> 48ull) != 0, "invalid address, over 48bit range.\n", 0);
999 CHK_CONDITION((addr >> (memzone == MEMZONE_SYS ? 40ull : 41ull)) != 0, "invalid address, over memory zone range.\n", 0);
1000 CHK_CONDITION((addr % alignment) != 0, "invalid address, not meet aligment requirement.\n", 0);
1001
1002 return addr;
1003 }
1004
1005 static void
mos_gem_bo_vma_free(struct mos_bufmgr * bufmgr,uint64_t address,uint64_t size)1006 mos_gem_bo_vma_free(struct mos_bufmgr *bufmgr,
1007 uint64_t address,
1008 uint64_t size)
1009 {
1010 CHK_CONDITION(bufmgr == nullptr, "nullptr bufmgr.\n", );
1011 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1012
1013 CHK_CONDITION(address == 0ull, "invalid address.\n", );
1014 enum mos_memory_zone memzone = mos_gem_bo_memzone_for_address(address);
1015 mos_vma_heap_free(&bufmgr_gem->vma_heap[memzone], address, size);
1016 }
1017
1018 drm_export struct mos_linux_bo *
mos_gem_bo_alloc_internal(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc * alloc)1019 mos_gem_bo_alloc_internal(struct mos_bufmgr *bufmgr,
1020 struct mos_drm_bo_alloc *alloc)
1021 {
1022 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1023 struct mos_bo_gem *bo_gem;
1024 unsigned int page_size = getpagesize();
1025 static bool support_pat_index = true;
1026 int ret;
1027 struct mos_gem_bo_bucket *bucket;
1028 bool alloc_from_cache;
1029 unsigned long bo_size;
1030 bool for_render = false;
1031
1032 if (alloc->ext.flags & BO_ALLOC_FOR_RENDER)
1033 for_render = true;
1034
1035 /* Round the allocated size up to a power of two number of pages. */
1036 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, alloc->size);
1037
1038 /* If we don't have caching at this size, don't actually round the
1039 * allocation up.
1040 */
1041 if (bucket == nullptr) {
1042 bo_size = alloc->size;
1043 if (bo_size < page_size)
1044 bo_size = page_size;
1045 } else {
1046 bo_size = bucket->size;
1047 }
1048 if (!support_pat_index)
1049 {
1050 /* For old kernel without pat index support,
1051 * We need to reset pat_index for bo reuse policy
1052 */
1053 alloc->ext.pat_index = PAT_INDEX_INVALID;
1054 }
1055 pthread_mutex_lock(&bufmgr_gem->lock);
1056 /* Get a buffer out of the cache if available */
1057 retry:
1058 alloc_from_cache = false;
1059 if (bucket != nullptr && !DRMLISTEMPTY(&bucket->head)) {
1060 if (for_render) {
1061 /* Allocate new render-target BOs from the tail (MRU)
1062 * of the list, as it will likely be hot in the GPU
1063 * cache and in the aperture for us.
1064 */
1065 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1066 bucket->head.prev, head);
1067 DRMLISTDEL(&bo_gem->head);
1068 alloc_from_cache = true;
1069 bo_gem->bo.align = alloc->alignment;
1070 } else {
1071 assert(alloc->alignment == 0);
1072 /* For non-render-target BOs (where we're probably
1073 * going to map it first thing in order to fill it
1074 * with data), check if the last BO in the cache is
1075 * unbusy, and only reuse in that case. Otherwise,
1076 * allocating a new buffer is probably faster than
1077 * waiting for the GPU to finish.
1078 */
1079 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1080 bucket->head.next, head);
1081 if (!mos_gem_bo_busy(&bo_gem->bo)) {
1082 alloc_from_cache = true;
1083 DRMLISTDEL(&bo_gem->head);
1084 }
1085 }
1086
1087 if (alloc_from_cache) {
1088 if (!mos_gem_bo_madvise_internal
1089 (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {
1090 mos_gem_bo_free(&bo_gem->bo);
1091 mos_gem_bo_cache_purge_bucket(bufmgr_gem,
1092 bucket);
1093 goto retry;
1094 }
1095 if (bo_gem->pat_index != alloc->ext.pat_index)
1096 {
1097 mos_gem_bo_free(&bo_gem->bo);
1098 goto retry;
1099 }
1100 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1101 alloc->ext.tiling_mode,
1102 alloc->stride)) {
1103 mos_gem_bo_free(&bo_gem->bo);
1104 goto retry;
1105 }
1106 if (bufmgr_gem->has_lmem && mos_gem_bo_check_mem_region_internal(&bo_gem->bo, alloc->ext.mem_type)) {
1107 mos_gem_bo_free(&bo_gem->bo);
1108 goto retry;
1109 }
1110 }
1111 }
1112 pthread_mutex_unlock(&bufmgr_gem->lock);
1113
1114 if (!alloc_from_cache) {
1115
1116 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1117 if (!bo_gem)
1118 return nullptr;
1119
1120 bo_gem->bo.size = bo_size;
1121 bo_gem->mem_region = I915_MEMORY_CLASS_SYSTEM;
1122 bo_gem->pat_index = PAT_INDEX_INVALID;
1123 bo_gem->cpu_cacheable = true;
1124
1125 if (bufmgr_gem->has_lmem &&
1126 (alloc->ext.mem_type == MOS_MEMPOOL_VIDEOMEMORY || alloc->ext.mem_type == MOS_MEMPOOL_DEVICEMEMORY)) {
1127 struct drm_i915_gem_memory_class_instance mem_region;
1128 memclear(mem_region);
1129 mem_region.memory_class = I915_MEMORY_CLASS_DEVICE;
1130 mem_region.memory_instance = 0;
1131
1132 struct drm_i915_gem_create_ext_memory_regions regions;
1133 memclear(regions);
1134 regions.base.name = I915_GEM_CREATE_EXT_MEMORY_REGIONS;
1135 regions.num_regions = 1;
1136 regions.regions = (uintptr_t)(&mem_region);
1137
1138 struct drm_i915_gem_create_ext create;
1139 memclear(create);
1140 create.size = bo_size;
1141 create.extensions = (uintptr_t)(®ions);
1142
1143 ret = drmIoctl(bufmgr_gem->fd,
1144 DRM_IOCTL_I915_GEM_CREATE_EXT,
1145 &create);
1146 bo_gem->gem_handle = create.handle;
1147 bo_gem->bo.handle = bo_gem->gem_handle;
1148 bo_gem->mem_region = I915_MEMORY_CLASS_DEVICE;
1149 }
1150 else
1151 {
1152 ret = -EINVAL;
1153 if (support_pat_index && alloc->ext.pat_index != PAT_INDEX_INVALID)
1154 {
1155 struct drm_i915_gem_create_ext_set_pat set_pat_ext;
1156 memclear(set_pat_ext);
1157 set_pat_ext.base.name = I915_GEM_CREATE_EXT_SET_PAT;
1158 set_pat_ext.pat_index = alloc->ext.pat_index;
1159
1160 struct drm_i915_gem_create_ext create;
1161 memclear(create);
1162 create.size = bo_size;
1163 create.extensions = (uintptr_t)(&set_pat_ext);
1164 ret = drmIoctl(bufmgr_gem->fd,
1165 DRM_IOCTL_I915_GEM_CREATE_EXT,
1166 &create);
1167 bo_gem->gem_handle = create.handle;
1168 bo_gem->bo.handle = bo_gem->gem_handle;
1169 bo_gem->pat_index = alloc->ext.pat_index;
1170 if (ret != 0)
1171 {
1172 /* For old kernel without pat_index support,
1173 * DRM_IOCTL_I915_GEM_CREATE_EXT with unknown
1174 * set_pat_ext extension will return -EINVAL
1175 * support_pat_index need to be set false.
1176 */
1177 support_pat_index = false;
1178 }
1179 }
1180 if (ret != 0)
1181 {
1182 struct drm_i915_gem_create create;
1183 memclear(create);
1184 create.size = bo_size;
1185 ret = drmIoctl(bufmgr_gem->fd,
1186 DRM_IOCTL_I915_GEM_CREATE,
1187 &create);
1188 bo_gem->gem_handle = create.handle;
1189 bo_gem->bo.handle = bo_gem->gem_handle;
1190 bo_gem->pat_index = PAT_INDEX_INVALID;
1191 }
1192 }
1193 if (ret != 0) {
1194 free(bo_gem);
1195 return nullptr;
1196 }
1197 bo_gem->bo.bufmgr = bufmgr;
1198 bo_gem->bo.align = alloc->alignment;
1199
1200 bo_gem->tiling_mode = I915_TILING_NONE;
1201 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1202 bo_gem->stride = 0;
1203 if (bufmgr_gem->mem_profiler_fd != -1)
1204 {
1205 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CREATE, %d, %d, %lu, %d, %s\n", getpid(), bo_gem->bo.handle, bo_gem->bo.size,bo_gem->mem_region, alloc->name);
1206 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1207 if (ret == -1)
1208 {
1209 MOS_DBG("Failed to write to %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
1210 }
1211 }
1212
1213 /* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized
1214 list (vma_list), so better set the list head here */
1215 DRMINITLISTHEAD(&bo_gem->name_list);
1216 if (mos_gem_bo_set_tiling_internal(&bo_gem->bo,
1217 alloc->ext.tiling_mode,
1218 alloc->stride)) {
1219 mos_gem_bo_free(&bo_gem->bo);
1220 return nullptr;
1221 }
1222 }
1223
1224 bo_gem->name = alloc->name;
1225 atomic_set(&bo_gem->refcount, 1);
1226 bo_gem->validate_index = -1;
1227 bo_gem->reloc_tree_fences = 0;
1228 bo_gem->used_as_reloc_target = false;
1229 bo_gem->has_error = false;
1230 bo_gem->reusable = true;
1231 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1232
1233 if (bo_gem->pat_index != PAT_INDEX_INVALID)
1234 {
1235 bo_gem->cpu_cacheable = alloc->ext.cpu_cacheable;
1236 }
1237
1238 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alloc->alignment);
1239
1240 if (bufmgr_gem->use_softpin)
1241 {
1242 mos_bo_set_softpin(&bo_gem->bo);
1243 }
1244
1245 MOS_DBG("bo_create: buf %d (%s) %ldb\n",
1246 bo_gem->gem_handle, bo_gem->name, alloc->size);
1247
1248 return &bo_gem->bo;
1249 }
1250
1251 static struct mos_linux_bo *
mos_gem_bo_alloc(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc * alloc)1252 mos_gem_bo_alloc(struct mos_bufmgr *bufmgr,
1253 struct mos_drm_bo_alloc *alloc)
1254 {
1255 alloc->ext.flags = 0;
1256 alloc->alignment = 0;
1257 alloc->stride = 0;
1258 return mos_gem_bo_alloc_internal(bufmgr, alloc);
1259 }
1260
1261 static struct mos_linux_bo *
mos_gem_bo_alloc_tiled(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_tiled * alloc_tiled)1262 mos_gem_bo_alloc_tiled(struct mos_bufmgr *bufmgr,
1263 struct mos_drm_bo_alloc_tiled *alloc_tiled)
1264 {
1265 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
1266 unsigned long size, stride;
1267 uint32_t tiling;
1268
1269 do {
1270 unsigned long aligned_y, height_alignment;
1271
1272 tiling = alloc_tiled->ext.tiling_mode;
1273
1274 /* If we're tiled, our allocations are in 8 or 32-row blocks,
1275 * so failure to align our height means that we won't allocate
1276 * enough pages.
1277 *
1278 * If we're untiled, we still have to align to 2 rows high
1279 * because the data port accesses 2x2 blocks even if the
1280 * bottom row isn't to be rendered, so failure to align means
1281 * we could walk off the end of the GTT and fault. This is
1282 * documented on 965, and may be the case on older chipsets
1283 * too so we try to be careful.
1284 */
1285 aligned_y = alloc_tiled->y;
1286 height_alignment = 2;
1287
1288 if (tiling == I915_TILING_X
1289 || (IS_915(bufmgr_gem->pci_device)
1290 && tiling == I915_TILING_Y))
1291 height_alignment = 8;
1292 else if (tiling == I915_TILING_Y)
1293 height_alignment = 32;
1294 aligned_y = ALIGN(alloc_tiled->y, height_alignment);
1295
1296 stride = alloc_tiled->x * alloc_tiled->cpp;
1297 stride = mos_gem_bo_tile_pitch(bufmgr_gem, stride, &alloc_tiled->ext.tiling_mode);
1298 size = stride * aligned_y;
1299 size = mos_gem_bo_tile_size(bufmgr_gem, size, &alloc_tiled->ext.tiling_mode);
1300 } while (alloc_tiled->ext.tiling_mode != tiling);
1301 alloc_tiled->pitch = stride;
1302
1303 if (tiling == I915_TILING_NONE)
1304 stride = 0;
1305
1306 struct mos_drm_bo_alloc alloc;
1307 alloc.name = alloc_tiled->name;
1308 alloc.size = size;
1309 alloc.stride = stride;
1310 alloc.ext = alloc_tiled->ext;
1311 return mos_gem_bo_alloc_internal(bufmgr, &alloc);
1312 }
1313
1314 static struct mos_linux_bo *
mos_gem_bo_alloc_userptr(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_userptr * alloc_uptr)1315 mos_gem_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1316 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1317 {
1318 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1319 struct mos_bo_gem *bo_gem;
1320 int ret;
1321 struct drm_i915_gem_userptr userptr;
1322
1323 /* Tiling with userptr surfaces is not supported
1324 * on all hardware so refuse it for time being.
1325 */
1326 if (alloc_uptr->tiling_mode != I915_TILING_NONE)
1327 return nullptr;
1328
1329 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1330 if (!bo_gem)
1331 return nullptr;
1332
1333 bo_gem->bo.size = alloc_uptr->size;
1334
1335 memclear(userptr);
1336 userptr.user_ptr = (__u64)((unsigned long)alloc_uptr->addr);
1337 userptr.user_size = alloc_uptr->size;
1338 userptr.flags = 0;
1339
1340 ret = drmIoctl(bufmgr_gem->fd,
1341 DRM_IOCTL_I915_GEM_USERPTR,
1342 &userptr);
1343 if (ret != 0) {
1344 MOS_DBG("bo_create_userptr: "
1345 "ioctl failed with user ptr %p size 0x%lx, "
1346 "user flags 0x%lx\n", alloc_uptr->addr, alloc_uptr->size, alloc_uptr->flags);
1347 free(bo_gem);
1348 return nullptr;
1349 }
1350
1351 bo_gem->gem_handle = userptr.handle;
1352 bo_gem->bo.handle = bo_gem->gem_handle;
1353 bo_gem->bo.bufmgr = bufmgr;
1354 bo_gem->is_userptr = true;
1355 bo_gem->pat_index = PAT_INDEX_INVALID;
1356 bo_gem->cpu_cacheable = true;
1357 #ifdef __cplusplus
1358 bo_gem->bo.virt = alloc_uptr->addr;
1359 #else
1360 bo_gem->bo.virtual = alloc_uptr->addr;
1361 #endif
1362 /* Save the address provided by user */
1363 bo_gem->user_virtual = alloc_uptr->addr;
1364 bo_gem->tiling_mode = I915_TILING_NONE;
1365 bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
1366 bo_gem->stride = 0;
1367
1368 DRMINITLISTHEAD(&bo_gem->name_list);
1369
1370 bo_gem->name = alloc_uptr->name;
1371 atomic_set(&bo_gem->refcount, 1);
1372 bo_gem->validate_index = -1;
1373 bo_gem->reloc_tree_fences = 0;
1374 bo_gem->used_as_reloc_target = false;
1375 bo_gem->has_error = false;
1376 bo_gem->reusable = false;
1377 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1378
1379 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1380
1381 if (bufmgr_gem->use_softpin)
1382 {
1383 mos_bo_set_softpin(&bo_gem->bo);
1384 }
1385
1386 MOS_DBG("bo_create_userptr: "
1387 "ptr %p buf %d (%s) size %ldb, stride 0x%x, tile mode %d\n",
1388 alloc_uptr->addr, bo_gem->gem_handle, bo_gem->name,
1389 alloc_uptr->size, alloc_uptr->stride, alloc_uptr->tiling_mode);
1390
1391 return &bo_gem->bo;
1392 }
1393
1394 static bool
has_userptr(struct mos_bufmgr_gem * bufmgr_gem)1395 has_userptr(struct mos_bufmgr_gem *bufmgr_gem)
1396 {
1397 int ret;
1398 void *ptr;
1399 long pgsz;
1400 struct drm_i915_gem_userptr userptr;
1401
1402 pgsz = sysconf(_SC_PAGESIZE);
1403 assert(pgsz > 0);
1404
1405 ret = posix_memalign(&ptr, pgsz, pgsz);
1406 if (ret) {
1407 MOS_DBG("Failed to get a page (%ld) for userptr detection!\n",
1408 pgsz);
1409 return false;
1410 }
1411
1412 memclear(userptr);
1413 userptr.user_ptr = (__u64)(unsigned long)ptr;
1414 userptr.user_size = pgsz;
1415
1416 retry:
1417 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_USERPTR, &userptr);
1418 if (ret) {
1419 if (errno == ENODEV && userptr.flags == 0) {
1420 userptr.flags = I915_USERPTR_UNSYNCHRONIZED;
1421 goto retry;
1422 }
1423 free(ptr);
1424 return false;
1425 }
1426
1427 /* We don't release the userptr bo here as we want to keep the
1428 * kernel mm tracking alive for our lifetime. The first time we
1429 * create a userptr object the kernel has to install a mmu_notifer
1430 * which is a heavyweight operation (e.g. it requires taking all
1431 * mm_locks and stop_machine()).
1432 */
1433
1434 bufmgr_gem->userptr_active.ptr = ptr;
1435 bufmgr_gem->userptr_active.handle = userptr.handle;
1436 return true;
1437 }
1438
1439 static struct mos_linux_bo *
check_bo_alloc_userptr(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_userptr * alloc_uptr)1440 check_bo_alloc_userptr(struct mos_bufmgr *bufmgr,
1441 struct mos_drm_bo_alloc_userptr *alloc_uptr)
1442 {
1443 if (has_userptr((struct mos_bufmgr_gem *)bufmgr))
1444 bufmgr->bo_alloc_userptr = mos_gem_bo_alloc_userptr;
1445 else
1446 bufmgr->bo_alloc_userptr = nullptr;
1447
1448 return mos_bo_alloc_userptr(bufmgr, alloc_uptr);
1449 }
1450
1451 /**
1452 * Returns a drm_intel_bo wrapping the given buffer object handle.
1453 *
1454 * This can be used when one application needs to pass a buffer object
1455 * to another.
1456 */
1457 static struct mos_linux_bo *
mos_bufmgr_bo_gem_create_from_name(struct mos_bufmgr * bufmgr,const char * name,unsigned int handle)1458 mos_bufmgr_bo_gem_create_from_name(struct mos_bufmgr *bufmgr,
1459 const char *name,
1460 unsigned int handle)
1461 {
1462 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
1463 struct mos_bo_gem *bo_gem;
1464 int ret;
1465 struct drm_gem_open open_arg;
1466 struct drm_i915_gem_get_tiling get_tiling;
1467 drmMMListHead *list;
1468
1469 /* At the moment most applications only have a few named bo.
1470 * For instance, in a DRI client only the render buffers passed
1471 * between X and the client are named. And since X returns the
1472 * alternating names for the front/back buffer a linear search
1473 * provides a sufficiently fast match.
1474 */
1475 pthread_mutex_lock(&bufmgr_gem->lock);
1476 for (list = bufmgr_gem->named.next;
1477 list != &bufmgr_gem->named;
1478 list = list->next) {
1479 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1480 if (bo_gem->global_name == handle) {
1481 mos_gem_bo_reference(&bo_gem->bo);
1482 pthread_mutex_unlock(&bufmgr_gem->lock);
1483 return &bo_gem->bo;
1484 }
1485 }
1486
1487 memclear(open_arg);
1488 open_arg.name = handle;
1489 ret = drmIoctl(bufmgr_gem->fd,
1490 DRM_IOCTL_GEM_OPEN,
1491 &open_arg);
1492 if (ret != 0) {
1493 MOS_DBG("Couldn't reference %s handle 0x%08x: %s\n",
1494 name, handle, strerror(errno));
1495 pthread_mutex_unlock(&bufmgr_gem->lock);
1496 return nullptr;
1497 }
1498 /* Now see if someone has used a prime handle to get this
1499 * object from the kernel before by looking through the list
1500 * again for a matching gem_handle
1501 */
1502 for (list = bufmgr_gem->named.next;
1503 list != &bufmgr_gem->named;
1504 list = list->next) {
1505 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
1506 if (bo_gem->gem_handle == open_arg.handle) {
1507 mos_gem_bo_reference(&bo_gem->bo);
1508 pthread_mutex_unlock(&bufmgr_gem->lock);
1509 return &bo_gem->bo;
1510 }
1511 }
1512
1513 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
1514 if (!bo_gem) {
1515 pthread_mutex_unlock(&bufmgr_gem->lock);
1516 return nullptr;
1517 }
1518
1519 bo_gem->bo.size = open_arg.size;
1520 bo_gem->bo.offset = 0;
1521 bo_gem->bo.offset64 = 0;
1522 #if defined(__cplusplus)
1523 bo_gem->bo.virt = nullptr;
1524 #else
1525 bo_gem->bo.virtual = nullptr;
1526 #endif
1527 bo_gem->bo.bufmgr = bufmgr;
1528 bo_gem->name = name;
1529 bo_gem->pat_index = PAT_INDEX_INVALID;
1530 bo_gem->cpu_cacheable = true;
1531 atomic_set(&bo_gem->refcount, 1);
1532 bo_gem->validate_index = -1;
1533 bo_gem->gem_handle = open_arg.handle;
1534 bo_gem->bo.handle = open_arg.handle;
1535 bo_gem->global_name = handle;
1536 bo_gem->reusable = false;
1537 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
1538
1539 memclear(get_tiling);
1540 if (bufmgr_gem->has_fence_reg) {
1541 get_tiling.handle = bo_gem->gem_handle;
1542 ret = drmIoctl(bufmgr_gem->fd,
1543 DRM_IOCTL_I915_GEM_GET_TILING,
1544 &get_tiling);
1545 if (ret != 0) {
1546 MOS_DBG("create_from_name: failed to get tiling: %s\n", strerror(errno));
1547 mos_gem_bo_unreference(&bo_gem->bo);
1548 pthread_mutex_unlock(&bufmgr_gem->lock);
1549 return nullptr;
1550 }
1551 }
1552 else
1553 {
1554 MOS_DBG("create_from_name: driver ignored to get tiling from kernel\n");
1555 }
1556
1557 bo_gem->tiling_mode = get_tiling.tiling_mode;
1558 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
1559 /* XXX stride is unknown */
1560
1561 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
1562
1563 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
1564 pthread_mutex_unlock(&bufmgr_gem->lock);
1565
1566 if (bufmgr_gem->use_softpin)
1567 {
1568 mos_bo_set_softpin(&bo_gem->bo);
1569 }
1570
1571 MOS_DBG("bo_create_from_handle: %d (%s)\n", handle, bo_gem->name);
1572
1573 return &bo_gem->bo;
1574 }
1575
1576 static void
mos_gem_bo_free(struct mos_linux_bo * bo)1577 mos_gem_bo_free(struct mos_linux_bo *bo)
1578 {
1579 struct mos_bufmgr_gem *bufmgr_gem = nullptr;
1580 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1581 struct drm_gem_close close;
1582 int ret;
1583
1584 CHK_CONDITION(bo_gem == nullptr, "bo_gem == nullptr\n", );
1585
1586 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1587
1588 CHK_CONDITION(bufmgr_gem == nullptr, "bufmgr_gem == nullptr\n", );
1589
1590 if (bo_gem->mem_virtual) {
1591 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, 0));
1592 drm_munmap(bo_gem->mem_virtual, bo_gem->bo.size);
1593 bo_gem->mem_virtual = nullptr;
1594 }
1595 if (bo_gem->gtt_virtual) {
1596 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, 0));
1597 drm_munmap(bo_gem->gtt_virtual, bo_gem->bo.size);
1598 bo_gem->gtt_virtual = nullptr;
1599 }
1600 if (bo_gem->mem_wc_virtual) {
1601 VG(VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, 0));
1602 drm_munmap(bo_gem->mem_wc_virtual, bo_gem->bo.size);
1603 bo_gem->mem_wc_virtual = nullptr;
1604 }
1605
1606 if(bufmgr_gem->bufmgr.bo_wait_rendering && mos_gem_bo_busy(bo))
1607 {
1608 bufmgr_gem->bufmgr.bo_wait_rendering(bo);
1609 }
1610
1611 /* Close this object */
1612 memclear(close);
1613 close.handle = bo_gem->gem_handle;
1614 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close);
1615 if (ret != 0) {
1616 MOS_DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
1617 bo_gem->gem_handle, bo_gem->name, strerror(errno));
1618 }
1619 if (bufmgr_gem->mem_profiler_fd != -1)
1620 {
1621 snprintf(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE, "GEM_CLOSE, %d, %d, %lu, %d\n", getpid(), bo->handle,bo->size,bo_gem->mem_region);
1622 ret = write(bufmgr_gem->mem_profiler_fd, bufmgr_gem->mem_profiler_buffer, strnlen(bufmgr_gem->mem_profiler_buffer, MEM_PROFILER_BUFFER_SIZE));
1623 if (ret == -1)
1624 {
1625 MOS_DBG("Failed to write to %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
1626 }
1627 }
1628
1629 if (bufmgr_gem->use_softpin)
1630 {
1631 /* Return the VMA for reuse */
1632 mos_gem_bo_vma_free(bo->bufmgr, bo->offset64, bo->size);
1633 }
1634
1635 free(bo);
1636 }
1637
1638 static void
mos_gem_bo_mark_mmaps_incoherent(struct mos_linux_bo * bo)1639 mos_gem_bo_mark_mmaps_incoherent(struct mos_linux_bo *bo)
1640 {
1641 #if HAVE_VALGRIND
1642 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1643
1644 if (bo_gem->mem_virtual)
1645 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_virtual, bo->size);
1646
1647 if (bo_gem->gtt_virtual)
1648 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->gtt_virtual, bo->size);
1649
1650 if (bo_gem->mem_wc_virtual)
1651 VALGRIND_MAKE_MEM_NOACCESS(bo_gem->mem_wc_virtual, bo->size);
1652 #endif
1653 }
1654
1655 /** Frees all cached buffers significantly older than @time. */
1656 static void
mos_gem_cleanup_bo_cache(struct mos_bufmgr_gem * bufmgr_gem,time_t time)1657 mos_gem_cleanup_bo_cache(struct mos_bufmgr_gem *bufmgr_gem, time_t time)
1658 {
1659 int i;
1660
1661 if (bufmgr_gem->time == time)
1662 return;
1663
1664 for (i = 0; i < bufmgr_gem->num_buckets; i++) {
1665 struct mos_gem_bo_bucket *bucket =
1666 &bufmgr_gem->cache_bucket[i];
1667
1668 while (!DRMLISTEMPTY(&bucket->head)) {
1669 struct mos_bo_gem *bo_gem;
1670
1671 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
1672 bucket->head.next, head);
1673 if (time - bo_gem->free_time <= 1)
1674 break;
1675
1676 DRMLISTDEL(&bo_gem->head);
1677
1678 mos_gem_bo_free(&bo_gem->bo);
1679 }
1680 }
1681
1682 bufmgr_gem->time = time;
1683 }
1684
1685 drm_export void
mos_gem_bo_unreference_final(struct mos_linux_bo * bo,time_t time)1686 mos_gem_bo_unreference_final(struct mos_linux_bo *bo, time_t time)
1687 {
1688 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1689 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1690 struct mos_gem_bo_bucket *bucket;
1691 int i;
1692
1693 /* Unreference all the target buffers */
1694 for (i = 0; i < bo_gem->reloc_count; i++) {
1695 if (bo_gem->reloc_target_info[i].bo != bo) {
1696 mos_gem_bo_unreference_locked_timed(bo_gem->
1697 reloc_target_info[i].bo,
1698 time);
1699 }
1700 }
1701 for (i = 0; i < bo_gem->softpin_target_count; i++)
1702 mos_gem_bo_unreference_locked_timed(bo_gem->softpin_target[i].bo,
1703 time);
1704 bo_gem->reloc_count = 0;
1705 bo_gem->used_as_reloc_target = false;
1706 bo_gem->softpin_target_count = 0;
1707 bo_gem->exec_async = false;
1708
1709 MOS_DBG("bo_unreference final: %d (%s)\n",
1710 bo_gem->gem_handle, bo_gem->name);
1711 bo_gem->pad_to_size = 0;
1712
1713 /* release memory associated with this object */
1714 if (bo_gem->reloc_target_info) {
1715 free(bo_gem->reloc_target_info);
1716 bo_gem->reloc_target_info = nullptr;
1717 }
1718 if (bo_gem->relocs) {
1719 free(bo_gem->relocs);
1720 bo_gem->relocs = nullptr;
1721 }
1722 if (bo_gem->softpin_target) {
1723 free(bo_gem->softpin_target);
1724 bo_gem->softpin_target = nullptr;
1725 bo_gem->max_softpin_target_count = 0;
1726 }
1727
1728 /* Clear any left-over mappings */
1729 if (bo_gem->map_count) {
1730 MOS_DBG("bo freed with non-zero map-count %d\n", bo_gem->map_count);
1731 bo_gem->map_count = 0;
1732 mos_gem_bo_mark_mmaps_incoherent(bo);
1733 }
1734
1735 DRMLISTDEL(&bo_gem->name_list);
1736
1737 bucket = mos_gem_bo_bucket_for_size(bufmgr_gem, bo->size);
1738 /* Put the buffer into our internal cache for reuse if we can. */
1739 if (bufmgr_gem->bo_reuse && bo_gem->reusable && bucket != nullptr &&
1740 mos_gem_bo_madvise_internal(bufmgr_gem, bo_gem,
1741 I915_MADV_DONTNEED)) {
1742 bo_gem->free_time = time;
1743
1744 bo_gem->name = nullptr;
1745 bo_gem->validate_index = -1;
1746
1747 DRMLISTADDTAIL(&bo_gem->head, &bucket->head);
1748 } else {
1749 mos_gem_bo_free(bo);
1750 }
1751 }
1752
mos_gem_bo_unreference_locked_timed(struct mos_linux_bo * bo,time_t time)1753 static void mos_gem_bo_unreference_locked_timed(struct mos_linux_bo *bo,
1754 time_t time)
1755 {
1756 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1757
1758 assert(atomic_read(&bo_gem->refcount) > 0);
1759 if (atomic_dec_and_test(&bo_gem->refcount))
1760 mos_gem_bo_unreference_final(bo, time);
1761 }
1762
mos_gem_bo_unreference(struct mos_linux_bo * bo)1763 static void mos_gem_bo_unreference(struct mos_linux_bo *bo)
1764 {
1765 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1766
1767 assert(atomic_read(&bo_gem->refcount) > 0);
1768
1769 if (atomic_add_unless(&bo_gem->refcount, -1, 1)) {
1770 struct mos_bufmgr_gem *bufmgr_gem =
1771 (struct mos_bufmgr_gem *) bo->bufmgr;
1772 struct timespec time;
1773
1774 clock_gettime(CLOCK_MONOTONIC, &time);
1775
1776 pthread_mutex_lock(&bufmgr_gem->lock);
1777
1778 if (atomic_dec_and_test(&bo_gem->refcount)) {
1779 mos_gem_bo_unreference_final(bo, time.tv_sec);
1780 mos_gem_cleanup_bo_cache(bufmgr_gem, time.tv_sec);
1781 }
1782
1783 pthread_mutex_unlock(&bufmgr_gem->lock);
1784 }
1785 }
1786
1787 static int
map_wc(struct mos_linux_bo * bo)1788 map_wc(struct mos_linux_bo *bo)
1789 {
1790 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1791 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1792 int ret;
1793
1794 if (bo_gem->is_userptr)
1795 return -EINVAL;
1796
1797 if (!bufmgr_gem->has_ext_mmap)
1798 return -EINVAL;
1799
1800 /* Get a mapping of the buffer if we haven't before. */
1801 if (bo_gem->mem_wc_virtual == nullptr && bufmgr_gem->has_mmap_offset) {
1802 struct drm_i915_gem_mmap_offset mmap_arg;
1803
1804 MOS_DBG("bo_map_wc: mmap_offset %d (%s), map_count=%d\n",
1805 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1806
1807 memclear(mmap_arg);
1808 mmap_arg.handle = bo_gem->gem_handle;
1809 /* To indicate the uncached virtual mapping to KMD */
1810 if (bufmgr_gem->has_lmem)
1811 {
1812 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1813 }
1814 else
1815 {
1816 mmap_arg.flags = I915_MMAP_OFFSET_WC;
1817 }
1818 ret = drmIoctl(bufmgr_gem->fd,
1819 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
1820 &mmap_arg);
1821 if (ret != 0) {
1822 ret = -errno;
1823 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1824 __FILE__, __LINE__, bo_gem->gem_handle,
1825 bo_gem->name, strerror(errno));
1826 return ret;
1827 }
1828
1829 /* and mmap it */
1830 bo_gem->mem_wc_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1831 MAP_SHARED, bufmgr_gem->fd,
1832 mmap_arg.offset);
1833 if (bo_gem->mem_wc_virtual == MAP_FAILED) {
1834 bo_gem->mem_wc_virtual = nullptr;
1835 ret = -errno;
1836 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1837 __FILE__, __LINE__,
1838 bo_gem->gem_handle, bo_gem->name,
1839 strerror(errno));
1840 }
1841 }
1842 else if (bo_gem->mem_wc_virtual == nullptr) {
1843 struct drm_i915_gem_mmap mmap_arg;
1844
1845 MOS_DBG("bo_map_wc: mmap %d (%s), map_count=%d\n",
1846 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1847
1848 memclear(mmap_arg);
1849 mmap_arg.handle = bo_gem->gem_handle;
1850 /* To indicate the uncached virtual mapping to KMD */
1851 mmap_arg.flags = I915_MMAP_WC;
1852 mmap_arg.size = bo->size;
1853 ret = drmIoctl(bufmgr_gem->fd,
1854 DRM_IOCTL_I915_GEM_MMAP,
1855 &mmap_arg);
1856 if (ret != 0) {
1857 ret = -errno;
1858 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1859 __FILE__, __LINE__, bo_gem->gem_handle,
1860 bo_gem->name, strerror(errno));
1861 return ret;
1862 }
1863 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
1864 bo_gem->mem_wc_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
1865 }
1866 #ifdef __cplusplus
1867 bo->virt = bo_gem->mem_wc_virtual;
1868 #else
1869 bo->virtual = bo_gem->mem_wc_virtual;
1870 #endif
1871
1872 MOS_DBG("bo_map_wc: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
1873 bo_gem->mem_wc_virtual);
1874
1875 return 0;
1876 }
1877
1878 /* To be used in a similar way to mmap_gtt */
1879 drm_export int
mos_gem_bo_map_wc(struct mos_linux_bo * bo)1880 mos_gem_bo_map_wc(struct mos_linux_bo *bo) {
1881 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1882 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1883 struct drm_i915_gem_set_domain set_domain;
1884 struct drm_i915_gem_wait wait;
1885 int ret;
1886
1887 pthread_mutex_lock(&bufmgr_gem->lock);
1888
1889 ret = map_wc(bo);
1890 if (ret) {
1891 pthread_mutex_unlock(&bufmgr_gem->lock);
1892 return ret;
1893 }
1894
1895 if (bufmgr_gem->has_lmem) {
1896 assert(bufmgr_gem->has_wait_timeout);
1897 memclear(wait);
1898 wait.bo_handle = bo_gem->gem_handle;
1899 wait.timeout_ns = -1; // infinite wait
1900 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
1901 if (ret == -1) {
1902 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
1903 __FILE__, __LINE__, errno);
1904 }
1905 } else {
1906 /* Now move it to the GTT domain so that the GPU and CPU
1907 * caches are flushed and the GPU isn't actively using the
1908 * buffer.
1909 *
1910 * The domain change is done even for the objects which
1911 * are not bounded. For them first the pages are acquired,
1912 * before the domain change.
1913 */
1914 memclear(set_domain);
1915 set_domain.handle = bo_gem->gem_handle;
1916 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
1917 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
1918 ret = drmIoctl(bufmgr_gem->fd,
1919 DRM_IOCTL_I915_GEM_SET_DOMAIN,
1920 &set_domain);
1921 if (ret != 0) {
1922 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
1923 __FILE__, __LINE__, bo_gem->gem_handle,
1924 strerror(errno));
1925 }
1926 }
1927
1928 mos_gem_bo_mark_mmaps_incoherent(bo);
1929 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_wc_virtual, bo->size));
1930 pthread_mutex_unlock(&bufmgr_gem->lock);
1931
1932 return 0;
1933 }
1934
mos_gem_bo_map(struct mos_linux_bo * bo,int write_enable)1935 drm_export int mos_gem_bo_map(struct mos_linux_bo *bo, int write_enable)
1936 {
1937 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
1938 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
1939 int ret;
1940
1941 if (bo_gem->is_userptr) {
1942 /* Return the same user ptr */
1943 #ifdef __cplusplus
1944 bo->virt = bo_gem->user_virtual;
1945 #else
1946 bo->virtual = bo_gem->user_virtual;
1947 #endif
1948 return 0;
1949 }
1950 /* If cpu cacheable is false, it means bo is Non-Coherent. */
1951 if (!bo_gem->cpu_cacheable)
1952 {
1953 return mos_gem_bo_map_wc(bo);
1954 }
1955
1956 pthread_mutex_lock(&bufmgr_gem->lock);
1957
1958 if (bufmgr_gem->has_mmap_offset) {
1959 struct drm_i915_gem_wait wait;
1960
1961 if (!bo_gem->mem_virtual) {
1962 struct drm_i915_gem_mmap_offset mmap_arg;
1963
1964 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
1965 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
1966
1967 memclear(mmap_arg);
1968 mmap_arg.handle = bo_gem->gem_handle;
1969 if (bufmgr_gem->has_lmem)
1970 {
1971 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
1972 }
1973 else
1974 {
1975 mmap_arg.flags = I915_MMAP_OFFSET_WB;
1976 }
1977 ret = drmIoctl(bufmgr_gem->fd,
1978 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
1979 &mmap_arg);
1980 if (ret != 0) {
1981 ret = -errno;
1982 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1983 __FILE__, __LINE__, bo_gem->gem_handle,
1984 bo_gem->name, strerror(errno));
1985 pthread_mutex_unlock(&bufmgr_gem->lock);
1986 return ret;
1987 }
1988
1989 /* and mmap it */
1990 bo_gem->mem_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
1991 MAP_SHARED, bufmgr_gem->fd,
1992 mmap_arg.offset);
1993 if (bo_gem->mem_virtual == MAP_FAILED) {
1994 bo_gem->mem_virtual = nullptr;
1995 ret = -errno;
1996 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
1997 __FILE__, __LINE__,
1998 bo_gem->gem_handle, bo_gem->name,
1999 strerror(errno));
2000 }
2001 }
2002
2003 assert(bufmgr_gem->has_wait_timeout);
2004 memclear(wait);
2005 wait.bo_handle = bo_gem->gem_handle;
2006 wait.timeout_ns = -1; // infinite wait
2007 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2008 if (ret == -1) {
2009 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2010 __FILE__, __LINE__, errno);
2011 }
2012 } else { /*!has_mmap_offset*/
2013 struct drm_i915_gem_set_domain set_domain;
2014
2015 if (!bo_gem->mem_virtual) {
2016 struct drm_i915_gem_mmap mmap_arg;
2017
2018 MOS_DBG("bo_map: %d (%s), map_count=%d\n",
2019 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2020
2021 memclear(mmap_arg);
2022 mmap_arg.handle = bo_gem->gem_handle;
2023 mmap_arg.size = bo->size;
2024 ret = drmIoctl(bufmgr_gem->fd,
2025 DRM_IOCTL_I915_GEM_MMAP,
2026 &mmap_arg);
2027 if (ret != 0) {
2028 ret = -errno;
2029 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
2030 __FILE__, __LINE__, bo_gem->gem_handle,
2031 bo_gem->name, strerror(errno));
2032 pthread_mutex_unlock(&bufmgr_gem->lock);
2033 return ret;
2034 }
2035 VG(VALGRIND_MALLOCLIKE_BLOCK(mmap_arg.addr_ptr, mmap_arg.size, 0, 1));
2036 bo_gem->mem_virtual = (void *)(uintptr_t) mmap_arg.addr_ptr;
2037 }
2038
2039 memclear(set_domain);
2040 set_domain.handle = bo_gem->gem_handle;
2041 set_domain.read_domains = I915_GEM_DOMAIN_CPU;
2042 if (write_enable)
2043 set_domain.write_domain = I915_GEM_DOMAIN_CPU;
2044 else
2045 set_domain.write_domain = 0;
2046 ret = drmIoctl(bufmgr_gem->fd,
2047 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2048 &set_domain);
2049 if (ret != 0) {
2050 MOS_DBG("%s:%d: Error setting to CPU domain %d: %s\n",
2051 __FILE__, __LINE__, bo_gem->gem_handle,
2052 strerror(errno));
2053 }
2054 }
2055 MOS_DBG("bo_map: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
2056 bo_gem->mem_virtual);
2057 #ifdef __cplusplus
2058 bo->virt = bo_gem->mem_virtual;
2059 #else
2060 bo->virtual = bo_gem->mem_virtual;
2061 #endif
2062
2063 if (write_enable)
2064 bo_gem->mapped_cpu_write = true;
2065
2066 mos_gem_bo_mark_mmaps_incoherent(bo);
2067 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->mem_virtual, bo->size));
2068 pthread_mutex_unlock(&bufmgr_gem->lock);
2069
2070 return 0;
2071 }
2072
2073 drm_export int
map_gtt(struct mos_linux_bo * bo)2074 map_gtt(struct mos_linux_bo *bo)
2075 {
2076 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2077 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2078 int ret;
2079
2080 if (bo_gem->is_userptr)
2081 return -EINVAL;
2082
2083 /* Get a mapping of the buffer if we haven't before. */
2084 if (bo_gem->gtt_virtual == nullptr) {
2085 __u64 offset = 0;
2086 if (bufmgr_gem->has_lmem) {
2087 struct drm_i915_gem_mmap_offset mmap_arg;
2088
2089 MOS_DBG("map_gtt: mmap_offset %d (%s), map_count=%d\n",
2090 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2091
2092 memclear(mmap_arg);
2093 mmap_arg.handle = bo_gem->gem_handle;
2094 mmap_arg.flags = I915_MMAP_OFFSET_FIXED;
2095
2096 /* Get the fake offset back... */
2097 ret = drmIoctl(bufmgr_gem->fd,
2098 DRM_IOCTL_I915_GEM_MMAP_OFFSET,
2099 &mmap_arg);
2100 offset = mmap_arg.offset;
2101 }
2102 else
2103 {
2104 struct drm_i915_gem_mmap_gtt mmap_arg;
2105
2106 MOS_DBG("bo_map_gtt: mmap %d (%s), map_count=%d\n",
2107 bo_gem->gem_handle, bo_gem->name, bo_gem->map_count);
2108
2109 memclear(mmap_arg);
2110 mmap_arg.handle = bo_gem->gem_handle;
2111
2112 /* Get the fake offset back... */
2113 ret = drmIoctl(bufmgr_gem->fd,
2114 DRM_IOCTL_I915_GEM_MMAP_GTT,
2115 &mmap_arg);
2116 offset = mmap_arg.offset;
2117 }
2118 if (ret != 0) {
2119 ret = -errno;
2120 MOS_DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
2121 __FILE__, __LINE__,
2122 bo_gem->gem_handle, bo_gem->name,
2123 strerror(errno));
2124 return ret;
2125 }
2126
2127 /* and mmap it */
2128 bo_gem->gtt_virtual = drm_mmap(0, bo->size, PROT_READ | PROT_WRITE,
2129 MAP_SHARED, bufmgr_gem->fd,
2130 offset);
2131 if (bo_gem->gtt_virtual == MAP_FAILED) {
2132 bo_gem->gtt_virtual = nullptr;
2133 ret = -errno;
2134 MOS_DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
2135 __FILE__, __LINE__,
2136 bo_gem->gem_handle, bo_gem->name,
2137 strerror(errno));
2138 return ret;
2139 }
2140 }
2141 #ifdef __cplusplus
2142 bo->virt = bo_gem->gtt_virtual;
2143 #else
2144 bo->virtual = bo_gem->gtt_virtual;
2145 #endif
2146
2147 MOS_DBG("bo_map_gtt: %d (%s) -> %p\n", bo_gem->gem_handle, bo_gem->name,
2148 bo_gem->gtt_virtual);
2149
2150 return 0;
2151 }
2152
2153 static int
mos_gem_bo_map_gtt(struct mos_linux_bo * bo)2154 mos_gem_bo_map_gtt(struct mos_linux_bo *bo)
2155 {
2156 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2157 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2158 struct drm_i915_gem_set_domain set_domain;
2159 struct drm_i915_gem_wait wait;
2160 int ret;
2161
2162 pthread_mutex_lock(&bufmgr_gem->lock);
2163
2164 ret = map_gtt(bo);
2165 if (ret) {
2166 pthread_mutex_unlock(&bufmgr_gem->lock);
2167 return ret;
2168 }
2169
2170 if (bufmgr_gem->has_lmem) {
2171 assert(bufmgr_gem->has_wait_timeout);
2172 memclear(wait);
2173 wait.bo_handle = bo_gem->gem_handle;
2174 wait.timeout_ns = -1; // infinite wait
2175 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2176 if (ret == -1) {
2177 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2178 __FILE__, __LINE__, errno);
2179 }
2180 } else {
2181 /* Now move it to the GTT domain so that the GPU and CPU
2182 * caches are flushed and the GPU isn't actively using the
2183 * buffer.
2184 *
2185 * The pagefault handler does this domain change for us when
2186 * it has unbound the BO from the GTT, but it's up to us to
2187 * tell it when we're about to use things if we had done
2188 * rendering and it still happens to be bound to the GTT.
2189 */
2190 memclear(set_domain);
2191 set_domain.handle = bo_gem->gem_handle;
2192 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2193 set_domain.write_domain = I915_GEM_DOMAIN_GTT;
2194 ret = drmIoctl(bufmgr_gem->fd,
2195 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2196 &set_domain);
2197 if (ret != 0) {
2198 MOS_DBG("%s:%d: Error setting domain %d: %s\n",
2199 __FILE__, __LINE__, bo_gem->gem_handle,
2200 strerror(errno));
2201 }
2202 }
2203 mos_gem_bo_mark_mmaps_incoherent(bo);
2204 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
2205 pthread_mutex_unlock(&bufmgr_gem->lock);
2206
2207 return 0;
2208 }
2209
2210 /**
2211 * Performs a mapping of the buffer object like the normal GTT
2212 * mapping, but avoids waiting for the GPU to be done reading from or
2213 * rendering to the buffer.
2214 *
2215 * This is used in the implementation of GL_ARB_map_buffer_range: The
2216 * user asks to create a buffer, then does a mapping, fills some
2217 * space, runs a drawing command, then asks to map it again without
2218 * synchronizing because it guarantees that it won't write over the
2219 * data that the GPU is busy using (or, more specifically, that if it
2220 * does write over the data, it acknowledges that rendering is
2221 * undefined).
2222 */
2223
2224 static int
mos_gem_bo_map_unsynchronized(struct mos_linux_bo * bo)2225 mos_gem_bo_map_unsynchronized(struct mos_linux_bo *bo)
2226 {
2227 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2228 #ifdef HAVE_VALGRIND
2229 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2230 #endif
2231 int ret;
2232
2233 /* If the CPU cache isn't coherent with the GTT, then use a
2234 * regular synchronized mapping. The problem is that we don't
2235 * track where the buffer was last used on the CPU side in
2236 * terms of drm_intel_bo_map vs drm_intel_gem_bo_map_gtt, so
2237 * we would potentially corrupt the buffer even when the user
2238 * does reasonable things.
2239 */
2240 if (!bufmgr_gem->has_llc)
2241 return mos_gem_bo_map_gtt(bo);
2242
2243 pthread_mutex_lock(&bufmgr_gem->lock);
2244
2245 ret = map_gtt(bo);
2246 if (ret == 0) {
2247 mos_gem_bo_mark_mmaps_incoherent(bo);
2248 VG(VALGRIND_MAKE_MEM_DEFINED(bo_gem->gtt_virtual, bo->size));
2249 }
2250
2251 pthread_mutex_unlock(&bufmgr_gem->lock);
2252
2253 return ret;
2254 }
2255
mos_gem_bo_unmap(struct mos_linux_bo * bo)2256 static int mos_gem_bo_unmap(struct mos_linux_bo *bo)
2257 {
2258 struct mos_bufmgr_gem *bufmgr_gem;
2259 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2260 int ret = 0;
2261
2262 if (bo == nullptr)
2263 return 0;
2264
2265 if (bo_gem->is_userptr)
2266 return 0;
2267
2268 bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2269
2270 pthread_mutex_lock(&bufmgr_gem->lock);
2271
2272 if (bo_gem->map_count <= 0) {
2273 MOS_DBG("attempted to unmap an unmapped bo\n");
2274 pthread_mutex_unlock(&bufmgr_gem->lock);
2275 /* Preserve the old behaviour of just treating this as a
2276 * no-op rather than reporting the error.
2277 */
2278 return 0;
2279 }
2280
2281 if (bo_gem->mapped_cpu_write) {
2282 struct drm_i915_gem_sw_finish sw_finish;
2283
2284 /* Cause a flush to happen if the buffer's pinned for
2285 * scanout, so the results show up in a timely manner.
2286 * Unlike GTT set domains, this only does work if the
2287 * buffer should be scanout-related.
2288 */
2289 memclear(sw_finish);
2290 sw_finish.handle = bo_gem->gem_handle;
2291 ret = drmIoctl(bufmgr_gem->fd,
2292 DRM_IOCTL_I915_GEM_SW_FINISH,
2293 &sw_finish);
2294 ret = ret == -1 ? -errno : 0;
2295
2296 bo_gem->mapped_cpu_write = false;
2297 }
2298
2299 /* We need to unmap after every innovation as we cannot track
2300 * an open vma for every bo as that will exhaasut the system
2301 * limits and cause later failures.
2302 */
2303 if (--bo_gem->map_count == 0) {
2304 mos_gem_bo_mark_mmaps_incoherent(bo);
2305 #ifdef __cplusplus
2306 bo->virt = nullptr;
2307 #else
2308 bo->virtual = nullptr;
2309 #endif
2310 }
2311 pthread_mutex_unlock(&bufmgr_gem->lock);
2312
2313 return ret;
2314 }
2315
2316 static int
mos_gem_bo_unmap_wc(struct mos_linux_bo * bo)2317 mos_gem_bo_unmap_wc(struct mos_linux_bo *bo)
2318 {
2319 return mos_gem_bo_unmap(bo);
2320 }
2321
2322 static int
mos_gem_bo_unmap_gtt(struct mos_linux_bo * bo)2323 mos_gem_bo_unmap_gtt(struct mos_linux_bo *bo)
2324 {
2325 return mos_gem_bo_unmap(bo);
2326 }
2327
2328 /** Waits for all GPU rendering with the object to have completed. */
2329 static void
mos_gem_bo_wait_rendering(struct mos_linux_bo * bo)2330 mos_gem_bo_wait_rendering(struct mos_linux_bo *bo)
2331 {
2332 mos_gem_bo_start_gtt_access(bo, 1);
2333 }
2334
2335 /**
2336 * Waits on a BO for the given amount of time.
2337 *
2338 * @bo: buffer object to wait for
2339 * @timeout_ns: amount of time to wait in nanoseconds.
2340 * If value is less than 0, an infinite wait will occur.
2341 *
2342 * Returns 0 if the wait was successful ie. the last batch referencing the
2343 * object has completed within the allotted time. Otherwise some negative return
2344 * value describes the error. Of particular interest is -ETIME when the wait has
2345 * failed to yield the desired result.
2346 *
2347 * Similar to drm_intel_gem_bo_wait_rendering except a timeout parameter allows
2348 * the operation to give up after a certain amount of time. Another subtle
2349 * difference is the internal locking semantics are different (this variant does
2350 * not hold the lock for the duration of the wait). This makes the wait subject
2351 * to a larger userspace race window.
2352 *
2353 * The implementation shall wait until the object is no longer actively
2354 * referenced within a batch buffer at the time of the call. The wait will
2355 * not guarantee that the buffer is re-issued via another thread, or an flinked
2356 * handle. Userspace must make sure this race does not occur if such precision
2357 * is important.
2358 *
2359 * Note that some kernels have broken the inifite wait for negative values
2360 * promise, upgrade to latest stable kernels if this is the case.
2361 */
2362 drm_export int
mos_gem_bo_wait(struct mos_linux_bo * bo,int64_t timeout_ns)2363 mos_gem_bo_wait(struct mos_linux_bo *bo, int64_t timeout_ns)
2364 {
2365
2366 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2367 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2368 struct drm_i915_gem_wait wait;
2369 int ret;
2370
2371 if (!bufmgr_gem->has_wait_timeout) {
2372 MOS_DBG("%s:%d: Timed wait is not supported. Falling back to "
2373 "infinite wait\n", __FILE__, __LINE__);
2374 if (timeout_ns) {
2375 mos_gem_bo_wait_rendering(bo);
2376 return 0;
2377 } else {
2378 return mos_gem_bo_busy(bo) ? -ETIME : 0;
2379 }
2380 }
2381
2382 memclear(wait);
2383 wait.bo_handle = bo_gem->gem_handle;
2384 wait.timeout_ns = timeout_ns;
2385 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2386 if (ret == -1)
2387 return -errno;
2388
2389 return ret;
2390 }
2391
2392 /**
2393 * Sets the object to the GTT read and possibly write domain, used by the X
2394 * 2D driver in the absence of kernel support to do drm_intel_gem_bo_map_gtt().
2395 *
2396 * In combination with drm_intel_gem_bo_pin() and manual fence management, we
2397 * can do tiled pixmaps this way.
2398 */
2399 static void
mos_gem_bo_start_gtt_access(struct mos_linux_bo * bo,int write_enable)2400 mos_gem_bo_start_gtt_access(struct mos_linux_bo *bo, int write_enable)
2401 {
2402 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2403 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2404 struct drm_i915_gem_set_domain set_domain;
2405 struct drm_i915_gem_wait wait;
2406 int ret;
2407
2408 if (bufmgr_gem->has_lmem) {
2409 assert(bufmgr_gem->has_wait_timeout);
2410 memclear(wait);
2411 wait.bo_handle = bo_gem->gem_handle;
2412 wait.timeout_ns = -1; // infinite wait
2413 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
2414 if (ret == -1) {
2415 MOS_DBG("%s:%d: DRM_IOCTL_I915_GEM_WAIT failed (%d)\n",
2416 __FILE__, __LINE__, errno);
2417 }
2418 } else {
2419 memclear(set_domain);
2420 set_domain.handle = bo_gem->gem_handle;
2421 set_domain.read_domains = I915_GEM_DOMAIN_GTT;
2422 set_domain.write_domain = write_enable ? I915_GEM_DOMAIN_GTT : 0;
2423 ret = drmIoctl(bufmgr_gem->fd,
2424 DRM_IOCTL_I915_GEM_SET_DOMAIN,
2425 &set_domain);
2426 if (ret != 0) {
2427 MOS_DBG("%s:%d: Error setting memory domains %d (%08x %08x): %s .\n",
2428 __FILE__, __LINE__, bo_gem->gem_handle,
2429 set_domain.read_domains, set_domain.write_domain,
2430 strerror(errno));
2431 }
2432 }
2433 }
2434
2435 static void
mos_bufmgr_cleanup_cache(struct mos_bufmgr_gem * bufmgr_gem)2436 mos_bufmgr_cleanup_cache(struct mos_bufmgr_gem *bufmgr_gem)
2437 {
2438 for (int i = 0; i < bufmgr_gem->num_buckets; i++) {
2439 struct mos_gem_bo_bucket *bucket =
2440 &bufmgr_gem->cache_bucket[i];
2441 struct mos_bo_gem *bo_gem;
2442
2443 while (!DRMLISTEMPTY(&bucket->head)) {
2444 bo_gem = DRMLISTENTRY(struct mos_bo_gem,
2445 bucket->head.next, head);
2446 DRMLISTDEL(&bo_gem->head);
2447
2448 mos_gem_bo_free(&bo_gem->bo);
2449 }
2450 bufmgr_gem->cache_bucket[i].size = 0;
2451 }
2452 bufmgr_gem->num_buckets = 0;
2453 }
2454
2455 static void
mos_bufmgr_gem_destroy(struct mos_bufmgr * bufmgr)2456 mos_bufmgr_gem_destroy(struct mos_bufmgr *bufmgr)
2457 {
2458 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
2459 struct drm_gem_close close_bo;
2460 int ret;
2461
2462 free(bufmgr_gem->exec2_objects);
2463 free(bufmgr_gem->exec_objects);
2464 free(bufmgr_gem->exec_bos);
2465 pthread_mutex_destroy(&bufmgr_gem->lock);
2466
2467 /* Free any cached buffer objects we were going to reuse */
2468 mos_bufmgr_cleanup_cache(bufmgr_gem);
2469
2470 /* Release userptr bo kept hanging around for optimisation. */
2471 if (bufmgr_gem->userptr_active.ptr) {
2472 memclear(close_bo);
2473 close_bo.handle = bufmgr_gem->userptr_active.handle;
2474 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_CLOSE, &close_bo);
2475 free(bufmgr_gem->userptr_active.ptr);
2476 if (ret)
2477 fprintf(stderr,
2478 "Failed to release test userptr object! (%d) "
2479 "i915 kernel driver may not be sane!\n", errno);
2480 }
2481
2482 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_SYS]);
2483 mos_vma_heap_finish(&bufmgr_gem->vma_heap[MEMZONE_DEVICE]);
2484
2485 if (bufmgr_gem->mem_profiler_fd != -1)
2486 {
2487 close(bufmgr_gem->mem_profiler_fd);
2488 }
2489
2490 free(bufmgr);
2491 }
2492
2493 static int
do_bo_emit_reloc(struct mos_linux_bo * bo,uint32_t offset,struct mos_linux_bo * target_bo,uint32_t target_offset,uint32_t read_domains,uint32_t write_domain,bool need_fence,uint64_t presumed_offset)2494 do_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2495 struct mos_linux_bo *target_bo, uint32_t target_offset,
2496 uint32_t read_domains, uint32_t write_domain,
2497 bool need_fence, uint64_t presumed_offset)
2498 {
2499 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2500 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2501 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2502
2503 if (bo_gem->has_error)
2504 return -ENOMEM;
2505
2506 if (target_bo_gem->has_error) {
2507 bo_gem->has_error = true;
2508 return -ENOMEM;
2509 }
2510
2511 /* Create a new relocation list if needed */
2512 if (bo_gem->relocs == nullptr && mos_setup_reloc_list(bo))
2513 return -ENOMEM;
2514
2515 /* Check overflow */
2516 assert(bo_gem->reloc_count < bufmgr_gem->max_relocs);
2517
2518 /* Check args */
2519 assert(offset <= bo->size - 4);
2520 assert((write_domain & (write_domain - 1)) == 0);
2521
2522 /* An object needing a fence is a tiled buffer, so it won't have
2523 * relocs to other buffers.
2524 */
2525 if (need_fence) {
2526 assert(target_bo_gem->reloc_count == 0);
2527 target_bo_gem->reloc_tree_fences = 1;
2528 }
2529
2530 /* Make sure that we're not adding a reloc to something whose size has
2531 * already been accounted for.
2532 */
2533 assert(!bo_gem->used_as_reloc_target);
2534 if (target_bo_gem != bo_gem) {
2535 target_bo_gem->used_as_reloc_target = true;
2536 bo_gem->reloc_tree_size += target_bo_gem->reloc_tree_size;
2537 bo_gem->reloc_tree_fences += target_bo_gem->reloc_tree_fences;
2538 }
2539
2540 int flags = 0;
2541 if (target_bo_gem->pad_to_size)
2542 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2543 if (target_bo_gem->use_48b_address_range)
2544 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2545 if (target_bo_gem->exec_async)
2546 flags |= EXEC_OBJECT_ASYNC;
2547 if (target_bo_gem->exec_capture)
2548 flags |= EXEC_OBJECT_CAPTURE;
2549
2550 if (target_bo != bo)
2551 mos_gem_bo_reference(target_bo);
2552
2553 bo_gem->reloc_target_info[bo_gem->reloc_count].bo = target_bo;
2554 bo_gem->reloc_target_info[bo_gem->reloc_count].flags = flags;
2555 bo_gem->relocs[bo_gem->reloc_count].offset = offset;
2556 bo_gem->relocs[bo_gem->reloc_count].delta = target_offset;
2557 bo_gem->relocs[bo_gem->reloc_count].target_handle =
2558 target_bo_gem->gem_handle;
2559 bo_gem->relocs[bo_gem->reloc_count].read_domains = read_domains;
2560 bo_gem->relocs[bo_gem->reloc_count].write_domain = write_domain;
2561 bo_gem->relocs[bo_gem->reloc_count].presumed_offset = presumed_offset;
2562 bo_gem->reloc_count++;
2563
2564 return 0;
2565 }
2566
2567 static void
mos_gem_bo_use_48b_address_range(struct mos_linux_bo * bo,uint32_t enable)2568 mos_gem_bo_use_48b_address_range(struct mos_linux_bo *bo, uint32_t enable)
2569 {
2570 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2571 bo_gem->use_48b_address_range = enable;
2572 }
2573
2574 static void
mos_gem_bo_set_object_async(struct mos_linux_bo * bo)2575 mos_gem_bo_set_object_async(struct mos_linux_bo *bo)
2576 {
2577 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2578 bo_gem->exec_async = true;
2579 }
2580
2581 static void
mos_gem_bo_set_exec_object_async(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo)2582 mos_gem_bo_set_exec_object_async(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
2583 {
2584 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2585 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2586 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2587 int i;
2588 for (i = 0; i < bo_gem->reloc_count; i++)
2589 {
2590 if (bo_gem->reloc_target_info[i].bo == target_bo)
2591 {
2592 bo_gem->reloc_target_info[i].flags |= EXEC_OBJECT_ASYNC;
2593 break;
2594 }
2595 }
2596
2597 for (i = 0; i < bo_gem->softpin_target_count; i++)
2598 {
2599 if (bo_gem->softpin_target[i].bo == target_bo)
2600 {
2601 bo_gem->softpin_target[i].flags |= EXEC_OBJECT_ASYNC;
2602 break;
2603 }
2604 }
2605 }
2606
2607 static void
mos_gem_bo_set_object_capture(struct mos_linux_bo * bo)2608 mos_gem_bo_set_object_capture(struct mos_linux_bo *bo)
2609 {
2610 // Do nothing if bo is nullptr
2611 if (bo == nullptr)
2612 {
2613 return;
2614 }
2615 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2616 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2617 if (bufmgr_gem != nullptr &&
2618 bo_gem != nullptr &&
2619 !bufmgr_gem->object_capture_disabled)
2620 {
2621 bo_gem->exec_capture = true;
2622 }
2623 }
2624
2625 static int
mos_gem_bo_add_softpin_target(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo,bool write_flag)2626 mos_gem_bo_add_softpin_target(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo, bool write_flag)
2627 {
2628 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2629 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2630 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
2631 if (bo_gem->has_error)
2632 return -ENOMEM;
2633
2634 if (target_bo_gem->has_error) {
2635 bo_gem->has_error = true;
2636 return -ENOMEM;
2637 }
2638
2639 if (!target_bo_gem->is_softpin)
2640 return -EINVAL;
2641 if (target_bo_gem == bo_gem)
2642 return -EINVAL;
2643
2644 if (bo_gem->softpin_target_count == bo_gem->max_softpin_target_count) {
2645 int max_softpin_target_count = bo_gem->max_softpin_target_count * 2;
2646
2647 /* initial softpin target count*/
2648 if (max_softpin_target_count == 0){
2649 max_softpin_target_count = INITIAL_SOFTPIN_TARGET_COUNT;
2650 }
2651
2652 bo_gem->softpin_target = (struct mos_softpin_target *)realloc(bo_gem->softpin_target, max_softpin_target_count *
2653 sizeof(struct mos_softpin_target));
2654 if (!bo_gem->softpin_target)
2655 return -ENOMEM;
2656
2657 bo_gem->max_softpin_target_count = max_softpin_target_count;
2658 }
2659
2660 int flags = EXEC_OBJECT_PINNED;
2661 if (target_bo_gem->pad_to_size)
2662 flags |= EXEC_OBJECT_PAD_TO_SIZE;
2663 if (target_bo_gem->use_48b_address_range)
2664 flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2665 if (target_bo_gem->exec_async)
2666 flags |= EXEC_OBJECT_ASYNC;
2667 if (target_bo_gem->exec_capture)
2668 flags |= EXEC_OBJECT_CAPTURE;
2669 if (write_flag)
2670 flags |= EXEC_OBJECT_WRITE;
2671
2672 bo_gem->softpin_target[bo_gem->softpin_target_count].bo = target_bo;
2673 bo_gem->softpin_target[bo_gem->softpin_target_count].flags = flags;
2674 mos_gem_bo_reference(target_bo);
2675 bo_gem->softpin_target_count++;
2676
2677 return 0;
2678 }
2679
2680 static mos_oca_exec_list_info*
mos_bufmgr_bo_get_softpin_targets_info(struct mos_linux_bo * bo,int * count)2681 mos_bufmgr_bo_get_softpin_targets_info(struct mos_linux_bo *bo, int *count)
2682 {
2683 if(bo == nullptr || count == nullptr)
2684 {
2685 return nullptr;
2686 }
2687 mos_oca_exec_list_info *info = nullptr;
2688 std::vector<int> bo_added;
2689 int counter = 0;
2690 int MAX_COUNT = 50;
2691 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2692 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2693 int softpin_target_count = bo_gem->softpin_target_count;
2694 if(softpin_target_count == 0 || softpin_target_count > MAX_COUNT)
2695 {
2696 return info;
2697 }
2698 info = (mos_oca_exec_list_info *)malloc((softpin_target_count + 1) * sizeof(mos_oca_exec_list_info));
2699 if(info == nullptr)
2700 {
2701 return info;
2702 }
2703
2704 for(int i = 0; i < softpin_target_count; i++)
2705 {
2706 /*note: set capture for each bo*/
2707 struct mos_softpin_target *target = (struct mos_softpin_target *)&bo_gem->softpin_target[i];
2708 struct mos_bo_gem *target_gem = (struct mos_bo_gem *)target->bo;
2709 if(std::find(bo_added.begin(), bo_added.end(), target->bo->handle) == bo_added.end())
2710 {
2711 info[counter].handle = target->bo->handle;
2712 info[counter].size = target->bo->size;
2713 info[counter].offset64 = target->bo->offset64;
2714
2715 if (!bufmgr_gem->object_capture_disabled)
2716 target->flags |= EXEC_OBJECT_CAPTURE;
2717
2718 info[counter].flags = target->flags;
2719 info[counter].mem_region = target_gem->mem_region;
2720 info[counter].is_batch = false;
2721 bo_added.push_back(target->bo->handle);
2722 counter++;
2723 }
2724 }
2725
2726 /*note: bo is cmd bo, also need to be added*/
2727 int bb_flags = 0;
2728 if (bo_gem->pad_to_size)
2729 bb_flags |= EXEC_OBJECT_PAD_TO_SIZE;
2730 if (bo_gem->use_48b_address_range)
2731 bb_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
2732 if (bo_gem->is_softpin)
2733 bb_flags |= EXEC_OBJECT_PINNED;
2734 if (bo_gem->exec_async)
2735 bb_flags |= EXEC_OBJECT_ASYNC;
2736 if (bo_gem->exec_capture)
2737 bb_flags |= EXEC_OBJECT_CAPTURE;
2738
2739 info[counter].handle = bo->handle;
2740 info[counter].size = bo->size;
2741 info[counter].offset64 = bo->offset64;
2742 info[counter].flags = bb_flags;
2743 info[counter].mem_region = bo_gem->mem_region;
2744 info[counter].is_batch = true;
2745 counter++;
2746
2747 *count = counter;
2748
2749 return info;
2750 }
2751
2752 static int
mos_gem_bo_pad_to_size(struct mos_linux_bo * bo,uint64_t pad_to_size)2753 mos_gem_bo_pad_to_size(struct mos_linux_bo *bo, uint64_t pad_to_size)
2754 {
2755 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2756
2757 if (pad_to_size && pad_to_size < bo->size)
2758 return -EINVAL;
2759
2760 bo_gem->pad_to_size = pad_to_size;
2761 return 0;
2762 }
2763
2764 static int
mos_gem_bo_emit_reloc(struct mos_linux_bo * bo,uint32_t offset,struct mos_linux_bo * target_bo,uint32_t target_offset,uint32_t read_domains,uint32_t write_domain,uint64_t presumed_offset)2765 mos_gem_bo_emit_reloc(struct mos_linux_bo *bo, uint32_t offset,
2766 struct mos_linux_bo *target_bo, uint32_t target_offset,
2767 uint32_t read_domains, uint32_t write_domain,
2768 uint64_t presumed_offset)
2769 {
2770 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2771
2772 return do_bo_emit_reloc(bo, offset, target_bo, target_offset,
2773 read_domains, write_domain,
2774 false,
2775 presumed_offset);
2776 }
2777
2778 /**
2779 * Removes existing relocation entries in the BO after "start".
2780 *
2781 * This allows a user to avoid a two-step process for state setup with
2782 * counting up all the buffer objects and doing a
2783 * drm_intel_bufmgr_check_aperture_space() before emitting any of the
2784 * relocations for the state setup. Instead, save the state of the
2785 * batchbuffer including drm_intel_gem_get_reloc_count(), emit all the
2786 * state, and then check if it still fits in the aperture.
2787 *
2788 * Any further drm_intel_bufmgr_check_aperture_space() queries
2789 * involving this buffer in the tree are undefined after this call.
2790 *
2791 * This also removes all softpinned targets being referenced by the BO.
2792 */
2793 static void
mos_gem_bo_clear_relocs(struct mos_linux_bo * bo,int start)2794 mos_gem_bo_clear_relocs(struct mos_linux_bo *bo, int start)
2795 {
2796 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
2797 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2798 int i;
2799 struct timespec time;
2800
2801 clock_gettime(CLOCK_MONOTONIC, &time);
2802
2803 assert(bo_gem->reloc_count >= start);
2804
2805 /* Unreference the cleared target buffers */
2806 pthread_mutex_lock(&bufmgr_gem->lock);
2807
2808 for (i = start; i < bo_gem->reloc_count; i++) {
2809 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->reloc_target_info[i].bo;
2810 if (&target_bo_gem->bo != bo) {
2811 bo_gem->reloc_tree_fences -= target_bo_gem->reloc_tree_fences;
2812 target_bo_gem->used_as_reloc_target = false;
2813 target_bo_gem->reloc_count = 0;
2814 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo,
2815 time.tv_sec);
2816 }
2817 }
2818 bo_gem->reloc_count = start;
2819
2820 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2821 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) bo_gem->softpin_target[i].bo;
2822 mos_gem_bo_unreference_locked_timed(&target_bo_gem->bo, time.tv_sec);
2823 }
2824 bo_gem->softpin_target_count = 0;
2825
2826 pthread_mutex_unlock(&bufmgr_gem->lock);
2827
2828 }
2829
2830 /**
2831 * Walk the tree of relocations rooted at BO and accumulate the list of
2832 * validations to be performed and update the relocation buffers with
2833 * index values into the validation list.
2834 */
2835 static void
mos_gem_bo_process_reloc(struct mos_linux_bo * bo)2836 mos_gem_bo_process_reloc(struct mos_linux_bo *bo)
2837 {
2838 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2839 int i;
2840
2841 if (bo_gem->relocs == nullptr)
2842 return;
2843
2844 for (i = 0; i < bo_gem->reloc_count; i++) {
2845 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2846
2847 if (target_bo == bo)
2848 continue;
2849
2850 mos_gem_bo_mark_mmaps_incoherent(bo);
2851
2852 /* Continue walking the tree depth-first. */
2853 mos_gem_bo_process_reloc(target_bo);
2854
2855 /* Add the target to the validate list */
2856 mos_add_validate_buffer(target_bo);
2857 }
2858 }
2859
2860 static void
mos_gem_bo_process_reloc2(struct mos_linux_bo * bo)2861 mos_gem_bo_process_reloc2(struct mos_linux_bo *bo)
2862 {
2863 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2864 int i;
2865
2866 if (bo_gem->relocs == nullptr && bo_gem->softpin_target == nullptr)
2867 return;
2868
2869 for (i = 0; i < bo_gem->reloc_count; i++) {
2870 struct mos_linux_bo *target_bo = bo_gem->reloc_target_info[i].bo;
2871
2872 if (target_bo == bo)
2873 continue;
2874
2875 mos_gem_bo_mark_mmaps_incoherent(bo);
2876
2877 /* Continue walking the tree depth-first. */
2878 mos_gem_bo_process_reloc2(target_bo);
2879
2880 /* Add the target to the validate list */
2881 mos_add_reloc_objects(bo_gem->reloc_target_info[i]);
2882 }
2883
2884 for (i = 0; i < bo_gem->softpin_target_count; i++) {
2885 struct mos_linux_bo *target_bo = bo_gem->softpin_target[i].bo;
2886
2887 if (target_bo == bo)
2888 continue;
2889
2890 mos_gem_bo_mark_mmaps_incoherent(bo);
2891 mos_gem_bo_process_reloc2(target_bo);
2892 mos_add_softpin_objects(bo_gem->softpin_target[i]);
2893 }
2894 }
2895
2896 static void
mos_update_buffer_offsets(struct mos_bufmgr_gem * bufmgr_gem)2897 mos_update_buffer_offsets(struct mos_bufmgr_gem *bufmgr_gem)
2898 {
2899 int i;
2900
2901 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2902 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2903 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
2904
2905 /* Update the buffer offset */
2906 if (bufmgr_gem->exec_objects[i].offset != bo->offset64) {
2907 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2908 bo_gem->gem_handle, bo_gem->name,
2909 upper_32_bits(bo->offset64),
2910 lower_32_bits(bo->offset64),
2911 upper_32_bits(bufmgr_gem->exec_objects[i].offset),
2912 lower_32_bits(bufmgr_gem->exec_objects[i].offset));
2913 bo->offset64 = bufmgr_gem->exec_objects[i].offset;
2914 bo->offset = bufmgr_gem->exec_objects[i].offset;
2915 }
2916 }
2917 }
2918
2919 static void
mos_update_buffer_offsets2(struct mos_bufmgr_gem * bufmgr_gem,mos_linux_context * ctx,mos_linux_bo * cmd_bo)2920 mos_update_buffer_offsets2 (struct mos_bufmgr_gem *bufmgr_gem, mos_linux_context *ctx, mos_linux_bo *cmd_bo)
2921 {
2922 int i;
2923
2924 for (i = 0; i < bufmgr_gem->exec_count; i++) {
2925 struct mos_linux_bo *bo = bufmgr_gem->exec_bos[i];
2926 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
2927
2928 /* Update the buffer offset */
2929 if (bufmgr_gem->exec2_objects[i].offset != bo->offset64) {
2930 /* If we're seeing softpinned object here it means that the kernel
2931 * has relocated our object... Indicating a programming error
2932 */
2933 assert(!bo_gem->is_softpin);
2934 MOS_DBG("BO %d (%s) migrated: 0x%08x %08x -> 0x%08x %08x\n",
2935 bo_gem->gem_handle, bo_gem->name,
2936 upper_32_bits(bo->offset64),
2937 lower_32_bits(bo->offset64),
2938 upper_32_bits(bufmgr_gem->exec2_objects[i].offset),
2939 lower_32_bits(bufmgr_gem->exec2_objects[i].offset));
2940 bo->offset64 = bufmgr_gem->exec2_objects[i].offset;
2941 bo->offset = bufmgr_gem->exec2_objects[i].offset;
2942 }
2943
2944 if(!bufmgr_gem->use_softpin)
2945 {
2946 if (cmd_bo != bo) {
2947 auto item_ctx = ctx->pOsContext->contextOffsetList.begin();
2948 for (; item_ctx != ctx->pOsContext->contextOffsetList.end(); item_ctx++) {
2949 if (item_ctx->intel_context == ctx && item_ctx->target_bo == bo) {
2950 item_ctx->offset64 = bo->offset64;
2951 break;
2952 }
2953 }
2954 if ( item_ctx == ctx->pOsContext->contextOffsetList.end()) {
2955 struct MOS_CONTEXT_OFFSET newContext = {ctx,
2956 bo,
2957 bo->offset64};
2958 ctx->pOsContext->contextOffsetList.push_back(newContext);
2959 }
2960 }
2961 }
2962 }
2963 }
2964
2965 drm_export int
do_exec2(struct mos_linux_bo * bo,int used,struct mos_linux_context * ctx,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int flags,int * fence)2966 do_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
2967 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
2968 unsigned int flags, int *fence
2969 )
2970 {
2971
2972 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo->bufmgr;
2973 struct drm_i915_gem_execbuffer2 execbuf;
2974 int ret = 0;
2975 int i;
2976
2977 if (to_bo_gem(bo)->has_error)
2978 return -ENOMEM;
2979
2980 switch (flags & 0x7) {
2981 default:
2982 return -EINVAL;
2983 case I915_EXEC_BLT:
2984 if (!bufmgr_gem->has_blt)
2985 return -EINVAL;
2986 break;
2987 case I915_EXEC_BSD:
2988 if (!bufmgr_gem->has_bsd)
2989 return -EINVAL;
2990 break;
2991 case I915_EXEC_VEBOX:
2992 if (!bufmgr_gem->has_vebox)
2993 return -EINVAL;
2994 break;
2995 case I915_EXEC_RENDER:
2996 case I915_EXEC_DEFAULT:
2997 break;
2998 }
2999
3000 pthread_mutex_lock(&bufmgr_gem->lock);
3001 /* Update indices and set up the validate list. */
3002 mos_gem_bo_process_reloc2(bo);
3003
3004 /* Add the batch buffer to the validation list. There are no relocations
3005 * pointing to it.
3006 */
3007 mos_add_validate_buffer2(bo, 0);
3008
3009 memclear(execbuf);
3010 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
3011 execbuf.buffer_count = bufmgr_gem->exec_count;
3012 execbuf.batch_start_offset = 0;
3013 execbuf.batch_len = used;
3014 execbuf.cliprects_ptr = (uintptr_t)cliprects;
3015 execbuf.num_cliprects = num_cliprects;
3016 execbuf.DR1 = 0;
3017 execbuf.DR4 = DR4;
3018 execbuf.flags = flags;
3019 if (ctx == nullptr)
3020 i915_execbuffer2_set_context_id(execbuf, 0);
3021 else
3022 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
3023 execbuf.rsvd2 = 0;
3024 if(flags & I915_EXEC_FENCE_SUBMIT)
3025 {
3026 execbuf.rsvd2 = *fence;
3027 }
3028 if(flags & I915_EXEC_FENCE_OUT)
3029 {
3030 execbuf.rsvd2 = -1;
3031 }
3032
3033 if (bufmgr_gem->no_exec)
3034 goto skip_execution;
3035
3036 ret = drmIoctl(bufmgr_gem->fd,
3037 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
3038 &execbuf);
3039 if (ret != 0) {
3040 ret = -errno;
3041 if (ret == -ENOSPC) {
3042 MOS_DBG("Execbuffer fails to pin. "
3043 "Estimate: %u. Actual: %u. Available: %u\n",
3044 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
3045 bufmgr_gem->exec_count),
3046 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
3047 bufmgr_gem->exec_count),
3048 (unsigned int) bufmgr_gem->gtt_size);
3049 }
3050 }
3051
3052 if (ctx != nullptr)
3053 {
3054 mos_update_buffer_offsets2(bufmgr_gem, ctx, bo);
3055 }
3056
3057 if(flags & I915_EXEC_FENCE_OUT)
3058 {
3059 *fence = execbuf.rsvd2 >> 32;
3060 }
3061
3062 skip_execution:
3063 if (bufmgr_gem->bufmgr.debug)
3064 mos_gem_dump_validation_list(bufmgr_gem);
3065
3066 for (i = 0; i < bufmgr_gem->exec_count; i++) {
3067 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[i]);
3068
3069 bo_gem->idle = false;
3070
3071 /* Disconnect the buffer from the validate list */
3072 bo_gem->validate_index = -1;
3073 bufmgr_gem->exec_bos[i] = nullptr;
3074 }
3075 bufmgr_gem->exec_count = 0;
3076 pthread_mutex_unlock(&bufmgr_gem->lock);
3077
3078 return ret;
3079 }
3080
3081 static int
do_exec3(struct mos_linux_bo ** bo,int _num_bo,struct mos_linux_context * ctx,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int _flags,int * fence)3082 do_exec3(struct mos_linux_bo **bo, int _num_bo, struct mos_linux_context *ctx,
3083 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3084 unsigned int _flags, int *fence
3085 )
3086 {
3087 uint64_t flags = _flags;
3088 uint64_t num_bo = _num_bo;
3089 if((bo == nullptr) || (ctx == nullptr) || (num_bo == 0))
3090 {
3091 return -EINVAL;
3092 }
3093
3094 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bo[0]->bufmgr;
3095 struct drm_i915_gem_execbuffer2 execbuf;
3096 int ret = 0;
3097 int i;
3098
3099 pthread_mutex_lock(&bufmgr_gem->lock);
3100
3101 struct mos_exec_info exec_info;
3102 memset(static_cast<void*>(&exec_info), 0, sizeof(exec_info));
3103 exec_info.batch_obj = (struct drm_i915_gem_exec_object2 *) calloc (num_bo, sizeof(struct drm_i915_gem_exec_object2));
3104 if(exec_info.batch_obj == nullptr)
3105 {
3106 ret = -ENOMEM;
3107 goto skip_execution;
3108 }
3109 exec_info.obj_remain_size = OBJ512_SIZE;
3110 exec_info.obj = (struct drm_i915_gem_exec_object2 *) calloc (OBJ512_SIZE, sizeof(struct drm_i915_gem_exec_object2));
3111 if(exec_info.obj == nullptr)
3112 {
3113 ret = -ENOMEM;
3114 goto skip_execution;
3115 }
3116
3117 for(i = 0; i < num_bo; i++)
3118 {
3119 if (to_bo_gem(bo[i])->has_error)
3120 {
3121 ret = -ENOMEM;
3122 goto skip_execution;
3123 }
3124
3125 /* Update indices and set up the validate list. */
3126 mos_gem_bo_process_reloc2(bo[i]);
3127
3128 /* Add the batch buffer to the validation list. There are no relocations
3129 * pointing to it.
3130 */
3131 mos_add_validate_buffer2(bo[i], 0);
3132
3133 if((bufmgr_gem->exec_count - 1 + num_bo) > exec_info.obj_remain_size)
3134 {
3135 // origin size + OBJ512_SIZE + obj_count + batch_count;
3136 uint32_t new_obj_size = exec_info.obj_count + exec_info.obj_remain_size + OBJ512_SIZE + bufmgr_gem->exec_count - 1 + num_bo;
3137 struct drm_i915_gem_exec_object2 *new_obj = (struct drm_i915_gem_exec_object2 *)realloc(exec_info.obj, new_obj_size * sizeof(struct drm_i915_gem_exec_object2));
3138 if(new_obj == nullptr)
3139 {
3140 ret = -ENOMEM;
3141 goto skip_execution;
3142 }
3143 exec_info.obj_remain_size = new_obj_size - exec_info.obj_count;
3144 exec_info.obj = new_obj;
3145 }
3146 if(0 == i)
3147 {
3148 uint32_t cp_size = (bufmgr_gem->exec_count - 1) * sizeof(struct drm_i915_gem_exec_object2);
3149 memcpy(exec_info.obj, bufmgr_gem->exec2_objects, cp_size);
3150 exec_info.obj_count += (bufmgr_gem->exec_count - 1);
3151 exec_info.obj_remain_size -= (bufmgr_gem->exec_count - 1);
3152 }
3153 else
3154 {
3155 for(int e2 = 0; e2 < bufmgr_gem->exec_count - 1; e2++)
3156 {
3157 int e1;
3158 for(e1 = 0; e1 < exec_info.obj_count; e1++)
3159 {
3160 // skip the duplicated bo if it is already in the list of exec_info.obj
3161 if(bufmgr_gem->exec2_objects[e2].handle == exec_info.obj[e1].handle)
3162 {
3163 break;
3164 }
3165 }
3166 //if no duplicated bo found, add it into list of exec_info.obj
3167 if(e1 == exec_info.obj_count)
3168 {
3169 exec_info.obj[exec_info.obj_count] = bufmgr_gem->exec2_objects[e2];
3170 exec_info.obj_count++;
3171 exec_info.obj_remain_size--;
3172 }
3173 }
3174 }
3175 memcpy(&exec_info.batch_obj[i], &bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1], sizeof(struct drm_i915_gem_exec_object2));
3176 exec_info.batch_count++;
3177 uint32_t reloc_count = bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1].relocation_count;
3178 uint32_t cp_size = (reloc_count * sizeof(struct drm_i915_gem_relocation_entry));
3179
3180 struct drm_i915_gem_relocation_entry* ptr_reloc = (struct drm_i915_gem_relocation_entry *)calloc(reloc_count,sizeof(struct drm_i915_gem_relocation_entry));
3181 if(ptr_reloc == nullptr)
3182 {
3183 ret = -ENOMEM;
3184 goto skip_execution;
3185 }
3186 memcpy(ptr_reloc, (struct drm_i915_gem_relocation_entry *)bufmgr_gem->exec2_objects[bufmgr_gem->exec_count - 1].relocs_ptr, cp_size);
3187
3188 exec_info.batch_obj[i].relocs_ptr = (uintptr_t)ptr_reloc;
3189 exec_info.batch_obj[i].relocation_count = reloc_count;
3190
3191 //clear bo
3192 if (bufmgr_gem->bufmgr.debug)
3193 {
3194 mos_gem_dump_validation_list(bufmgr_gem);
3195 }
3196
3197 for (int j = 0; j < bufmgr_gem->exec_count; j++) {
3198 struct mos_bo_gem *bo_gem = to_bo_gem(bufmgr_gem->exec_bos[j]);
3199
3200 if(bo_gem)
3201 {
3202 bo_gem->idle = false;
3203
3204 /* Disconnect the buffer from the validate list */
3205 bo_gem->validate_index = -1;
3206 bufmgr_gem->exec_bos[j] = nullptr;
3207 }
3208 }
3209 bufmgr_gem->exec_count = 0;
3210 }
3211
3212 //add back batch obj to the last position
3213 for(i = 0; i < num_bo; i++)
3214 {
3215 exec_info.obj[exec_info.obj_count] = exec_info.batch_obj[i];
3216 exec_info.obj_count++;
3217 exec_info.obj_remain_size--;
3218 }
3219
3220 //save previous ptr
3221 exec_info.pSavePreviousExec2Objects = bufmgr_gem->exec2_objects;
3222 bufmgr_gem->exec_count = exec_info.obj_count;
3223 bufmgr_gem->exec2_objects = exec_info.obj;
3224
3225 memclear(execbuf);
3226 execbuf.buffers_ptr = (uintptr_t)bufmgr_gem->exec2_objects;
3227 execbuf.buffer_count = bufmgr_gem->exec_count;
3228 execbuf.batch_start_offset = 0;
3229 execbuf.cliprects_ptr = (uintptr_t)cliprects;
3230 execbuf.num_cliprects = num_cliprects;
3231 execbuf.DR1 = 0;
3232 execbuf.DR4 = DR4;
3233 execbuf.flags = flags;
3234 if (ctx == nullptr)
3235 i915_execbuffer2_set_context_id(execbuf, 0);
3236 else
3237 i915_execbuffer2_set_context_id(execbuf, ctx->ctx_id);
3238 execbuf.rsvd2 = 0;
3239 if((flags & I915_EXEC_FENCE_SUBMIT) || (flags & I915_EXEC_FENCE_IN))
3240 {
3241 execbuf.rsvd2 = *fence;
3242 }
3243 else if(flags & I915_EXEC_FENCE_OUT)
3244 {
3245 execbuf.rsvd2 = -1;
3246 }
3247
3248 if (bufmgr_gem->no_exec)
3249 goto skip_execution;
3250
3251 ret = drmIoctl(bufmgr_gem->fd,
3252 DRM_IOCTL_I915_GEM_EXECBUFFER2_WR,
3253 &execbuf);
3254
3255 if (ret != 0) {
3256 ret = -errno;
3257 if (ret == -ENOSPC) {
3258 MOS_DBG("Execbuffer fails to pin. "
3259 "Estimate: %u. Actual: %u. Available: %u\n",
3260 mos_gem_estimate_batch_space(bufmgr_gem->exec_bos,
3261 bufmgr_gem->exec_count),
3262 mos_gem_compute_batch_space(bufmgr_gem->exec_bos,
3263 bufmgr_gem->exec_count),
3264 (unsigned int) bufmgr_gem->gtt_size);
3265 }
3266 }
3267
3268 bufmgr_gem->exec2_objects = exec_info.pSavePreviousExec2Objects;
3269
3270 if(flags & I915_EXEC_FENCE_OUT)
3271 {
3272 *fence = execbuf.rsvd2 >> 32;
3273 }
3274
3275 skip_execution:
3276 if (bufmgr_gem->bufmgr.debug)
3277 mos_gem_dump_validation_list(bufmgr_gem);
3278
3279 bufmgr_gem->exec_count = 0;
3280 if(exec_info.batch_obj)
3281 {
3282 for(i = 0; i < num_bo; i++)
3283 {
3284 mos_safe_free((struct drm_i915_gem_relocation_entry *)exec_info.batch_obj[i].relocs_ptr);
3285 }
3286 }
3287 mos_safe_free(exec_info.obj);
3288 mos_safe_free(exec_info.batch_obj);
3289 pthread_mutex_unlock(&bufmgr_gem->lock);
3290
3291 return ret;
3292 }
3293
3294 static int
mos_gem_bo_exec2(struct mos_linux_bo * bo,int used,drm_clip_rect_t * cliprects,int num_cliprects,int DR4)3295 mos_gem_bo_exec2(struct mos_linux_bo *bo, int used,
3296 drm_clip_rect_t *cliprects, int num_cliprects,
3297 int DR4)
3298 {
3299 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
3300 I915_EXEC_RENDER, nullptr);
3301 }
3302
3303 static int
mos_gem_bo_mrb_exec2(struct mos_linux_bo * bo,int used,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int flags)3304 mos_gem_bo_mrb_exec2(struct mos_linux_bo *bo, int used,
3305 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3306 unsigned int flags)
3307 {
3308 return do_exec2(bo, used, nullptr, cliprects, num_cliprects, DR4,
3309 flags, nullptr);
3310 }
3311
3312 static int
mos_gem_bo_context_exec2(struct mos_linux_bo * bo,int used,struct mos_linux_context * ctx,drm_clip_rect_t * cliprects,int num_cliprects,int DR4,unsigned int flags,int * fence)3313 mos_gem_bo_context_exec2(struct mos_linux_bo *bo, int used, struct mos_linux_context *ctx,
3314 drm_clip_rect_t *cliprects, int num_cliprects, int DR4,
3315 unsigned int flags, int *fence)
3316 {
3317 return do_exec2(bo, used, ctx, cliprects, num_cliprects, DR4,
3318 flags, fence);
3319 }
3320
3321 static int
mos_gem_bo_context_exec3(struct mos_linux_bo ** bo,int num_bo,struct mos_linux_context * ctx,struct drm_clip_rect * cliprects,int num_cliprects,int DR4,unsigned int flags,int * fence)3322 mos_gem_bo_context_exec3(struct mos_linux_bo **bo, int num_bo, struct mos_linux_context *ctx,
3323 struct drm_clip_rect *cliprects, int num_cliprects, int DR4,
3324 unsigned int flags, int *fence)
3325 {
3326 return do_exec3(bo, num_bo, ctx, cliprects, num_cliprects, DR4,
3327 flags, fence);
3328 }
3329
3330 static int
mos_gem_bo_check_mem_region_internal(struct mos_linux_bo * bo,int mem_type)3331 mos_gem_bo_check_mem_region_internal(struct mos_linux_bo *bo,
3332 int mem_type)
3333 {
3334 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3335
3336 // when re-cycle the gem_bo, need to keep the VA space is keeping consistent on memory type
3337 if (bo_gem->mem_region == I915_MEMORY_CLASS_SYSTEM &&
3338 (mem_type == MOS_MEMPOOL_VIDEOMEMORY || mem_type == MOS_MEMPOOL_DEVICEMEMORY))
3339 return -EINVAL;
3340
3341 if (bo_gem->mem_region == I915_MEMORY_CLASS_DEVICE &&
3342 (mem_type == MOS_MEMPOOL_SYSTEMMEMORY))
3343 return -EINVAL;
3344
3345 return 0;
3346 }
3347
3348 static int
mos_gem_bo_set_tiling_internal(struct mos_linux_bo * bo,uint32_t tiling_mode,uint32_t stride)3349 mos_gem_bo_set_tiling_internal(struct mos_linux_bo *bo,
3350 uint32_t tiling_mode,
3351 uint32_t stride)
3352 {
3353 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3354 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3355 struct drm_i915_gem_set_tiling set_tiling;
3356 int ret;
3357
3358 if (!bufmgr_gem->has_fence_reg)
3359 return 0;
3360
3361 if (bo_gem->global_name == 0 &&
3362 tiling_mode == bo_gem->tiling_mode &&
3363 stride == bo_gem->stride)
3364 return 0;
3365
3366 memset(&set_tiling, 0, sizeof(set_tiling));
3367 /* set_tiling is slightly broken and overwrites the
3368 * input on the error path, so we have to open code
3369 * rmIoctl.
3370 */
3371 set_tiling.handle = bo_gem->gem_handle;
3372 set_tiling.tiling_mode = tiling_mode;
3373 set_tiling.stride = stride;
3374
3375 ret = drmIoctl(bufmgr_gem->fd,
3376 DRM_IOCTL_I915_GEM_SET_TILING,
3377 &set_tiling);
3378 if (ret == -1)
3379 return -errno;
3380
3381 bo_gem->tiling_mode = set_tiling.tiling_mode;
3382 bo_gem->swizzle_mode = set_tiling.swizzle_mode;
3383 bo_gem->stride = set_tiling.stride;
3384 return 0;
3385 }
3386
3387 static int
mos_gem_bo_set_tiling(struct mos_linux_bo * bo,uint32_t * tiling_mode,uint32_t stride)3388 mos_gem_bo_set_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
3389 uint32_t stride)
3390 {
3391 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3392 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3393 int ret;
3394
3395 /* Tiling with userptr surfaces is not supported
3396 * on all hardware so refuse it for time being.
3397 */
3398 if (bo_gem->is_userptr)
3399 return -EINVAL;
3400
3401 /* Linear buffers have no stride. By ensuring that we only ever use
3402 * stride 0 with linear buffers, we simplify our code.
3403 */
3404 if (*tiling_mode == I915_TILING_NONE)
3405 stride = 0;
3406
3407 ret = mos_gem_bo_set_tiling_internal(bo, *tiling_mode, stride);
3408 if (ret == 0)
3409 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
3410 *tiling_mode = bo_gem->tiling_mode;
3411 return ret;
3412 }
3413
3414 static int
mos_gem_bo_get_tiling(struct mos_linux_bo * bo,uint32_t * tiling_mode,uint32_t * swizzle_mode)3415 mos_gem_bo_get_tiling(struct mos_linux_bo *bo, uint32_t * tiling_mode,
3416 uint32_t * swizzle_mode)
3417 {
3418 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3419
3420 *tiling_mode = bo_gem->tiling_mode;
3421 *swizzle_mode = bo_gem->swizzle_mode;
3422 return 0;
3423 }
3424
3425 static int
mos_gem_bo_set_softpin_offset(struct mos_linux_bo * bo,uint64_t offset)3426 mos_gem_bo_set_softpin_offset(struct mos_linux_bo *bo, uint64_t offset)
3427 {
3428 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3429
3430 bo_gem->is_softpin = true;
3431 bo->offset64 = offset;
3432 bo->offset = offset;
3433 return 0;
3434 }
3435
3436 static int
mos_gem_bo_set_softpin(MOS_LINUX_BO * bo)3437 mos_gem_bo_set_softpin(MOS_LINUX_BO *bo)
3438 {
3439 int ret = 0;
3440 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3441 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3442
3443 pthread_mutex_lock(&bufmgr_gem->lock);
3444 if (!mos_gem_bo_is_softpin(bo))
3445 {
3446 uint64_t alignment = (bufmgr_gem->softpin_va1Malign) ? PAGE_SIZE_1M : PAGE_SIZE_64K;
3447 uint64_t offset = mos_gem_bo_vma_alloc(bo->bufmgr, (enum mos_memory_zone)bo_gem->mem_region, bo->size, alignment);
3448 ret = mos_gem_bo_set_softpin_offset(bo, offset);
3449 }
3450 pthread_mutex_unlock(&bufmgr_gem->lock);
3451
3452 if (ret == 0)
3453 {
3454 ret = mos_bo_use_48b_address_range(bo, 1);
3455 }
3456
3457 return ret;
3458 }
3459
3460 static struct mos_linux_bo *
mos_gem_bo_create_from_prime(struct mos_bufmgr * bufmgr,struct mos_drm_bo_alloc_prime * alloc_prime)3461 mos_gem_bo_create_from_prime(struct mos_bufmgr *bufmgr, struct mos_drm_bo_alloc_prime *alloc_prime)
3462 {
3463 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
3464 int ret;
3465 uint32_t handle;
3466 struct mos_bo_gem *bo_gem;
3467 struct drm_i915_gem_get_tiling get_tiling;
3468 int prime_fd = alloc_prime->prime_fd;
3469 int size = alloc_prime->size;
3470 drmMMListHead *list;
3471
3472 pthread_mutex_lock(&bufmgr_gem->lock);
3473 ret = drmPrimeFDToHandle(bufmgr_gem->fd, prime_fd, &handle);
3474 if (ret) {
3475 MOS_DBG("create_from_prime: failed to obtain handle from fd: %s\n", strerror(errno));
3476 pthread_mutex_unlock(&bufmgr_gem->lock);
3477 return nullptr;
3478 }
3479
3480 /*
3481 * See if the kernel has already returned this buffer to us. Just as
3482 * for named buffers, we must not create two bo's pointing at the same
3483 * kernel object
3484 */
3485 for (list = bufmgr_gem->named.next;
3486 list != &bufmgr_gem->named;
3487 list = list->next) {
3488 bo_gem = DRMLISTENTRY(struct mos_bo_gem, list, name_list);
3489 if (bo_gem->gem_handle == handle) {
3490 mos_gem_bo_reference(&bo_gem->bo);
3491 pthread_mutex_unlock(&bufmgr_gem->lock);
3492 return &bo_gem->bo;
3493 }
3494 }
3495
3496 bo_gem = (struct mos_bo_gem *)calloc(1, sizeof(*bo_gem));
3497 if (!bo_gem) {
3498 pthread_mutex_unlock(&bufmgr_gem->lock);
3499 return nullptr;
3500 }
3501 /* Determine size of bo. The fd-to-handle ioctl really should
3502 * return the size, but it doesn't. If we have kernel 3.12 or
3503 * later, we can lseek on the prime fd to get the size. Older
3504 * kernels will just fail, in which case we fall back to the
3505 * provided (estimated or guess size). */
3506 ret = lseek(prime_fd, 0, SEEK_END);
3507 if (ret != -1)
3508 bo_gem->bo.size = ret;
3509 else
3510 bo_gem->bo.size = size;
3511
3512 bo_gem->bo.handle = handle;
3513 bo_gem->bo.bufmgr = bufmgr;
3514
3515 bo_gem->gem_handle = handle;
3516 bo_gem->pat_index = PAT_INDEX_INVALID;
3517 bo_gem->cpu_cacheable = true;
3518 atomic_set(&bo_gem->refcount, 1);
3519
3520 bo_gem->name = alloc_prime->name;
3521 bo_gem->validate_index = -1;
3522 bo_gem->reloc_tree_fences = 0;
3523 bo_gem->used_as_reloc_target = false;
3524 bo_gem->has_error = false;
3525 bo_gem->reusable = false;
3526 bo_gem->use_48b_address_range = bufmgr_gem->bufmgr.bo_use_48b_address_range ? true : false;
3527
3528 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3529 pthread_mutex_unlock(&bufmgr_gem->lock);
3530
3531 memclear(get_tiling);
3532 if(bufmgr_gem->has_fence_reg) {
3533 get_tiling.handle = bo_gem->gem_handle;
3534 ret = drmIoctl(bufmgr_gem->fd,
3535 DRM_IOCTL_I915_GEM_GET_TILING,
3536 &get_tiling);
3537 if (ret != 0) {
3538 MOS_DBG("create_from_prime: failed to get tiling: %s\n", strerror(errno));
3539 mos_gem_bo_unreference(&bo_gem->bo);
3540 return nullptr;
3541 }
3542 }
3543 else
3544 {
3545 MOS_DBG("create_from_prime: driver ignored to get tiling from kernel\n");
3546 }
3547
3548 bo_gem->tiling_mode = get_tiling.tiling_mode;
3549 bo_gem->swizzle_mode = get_tiling.swizzle_mode;
3550 /* XXX stride is unknown */
3551 mos_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, 0);
3552 if (bufmgr_gem->use_softpin)
3553 {
3554 mos_bo_set_softpin(&bo_gem->bo);
3555 }
3556
3557 return &bo_gem->bo;
3558 }
3559
3560 static int
mos_gem_bo_export_to_prime(struct mos_linux_bo * bo,int * prime_fd)3561 mos_gem_bo_export_to_prime(struct mos_linux_bo *bo, int *prime_fd)
3562 {
3563 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3564 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3565
3566 pthread_mutex_lock(&bufmgr_gem->lock);
3567 if (DRMLISTEMPTY(&bo_gem->name_list))
3568 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3569 pthread_mutex_unlock(&bufmgr_gem->lock);
3570
3571 if (drmPrimeHandleToFD(bufmgr_gem->fd, bo_gem->gem_handle,
3572 DRM_CLOEXEC, prime_fd) != 0)
3573 return -errno;
3574
3575 bo_gem->reusable = false;
3576
3577 return 0;
3578 }
3579
3580 static int
mos_gem_bo_flink(struct mos_linux_bo * bo,uint32_t * name)3581 mos_gem_bo_flink(struct mos_linux_bo *bo, uint32_t * name)
3582 {
3583 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bo->bufmgr;
3584 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3585 int ret;
3586
3587 if (!bo_gem->global_name) {
3588 struct drm_gem_flink flink;
3589
3590 memclear(flink);
3591 flink.handle = bo_gem->gem_handle;
3592
3593 pthread_mutex_lock(&bufmgr_gem->lock);
3594
3595 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_GEM_FLINK, &flink);
3596 if (ret != 0) {
3597 pthread_mutex_unlock(&bufmgr_gem->lock);
3598 return -errno;
3599 }
3600
3601 bo_gem->global_name = flink.name;
3602 bo_gem->reusable = false;
3603
3604 if (DRMLISTEMPTY(&bo_gem->name_list))
3605 DRMLISTADDTAIL(&bo_gem->name_list, &bufmgr_gem->named);
3606 pthread_mutex_unlock(&bufmgr_gem->lock);
3607 }
3608
3609 *name = bo_gem->global_name;
3610 return 0;
3611 }
3612
3613 /**
3614 * Enables unlimited caching of buffer objects for reuse.
3615 *
3616 * This is potentially very memory expensive, as the cache at each bucket
3617 * size is only bounded by how many buffers of that size we've managed to have
3618 * in flight at once.
3619 */
3620 static void
mos_gem_enable_reuse(struct mos_bufmgr * bufmgr)3621 mos_gem_enable_reuse(struct mos_bufmgr *bufmgr)
3622 {
3623 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *) bufmgr;
3624
3625 bufmgr_gem->bo_reuse = true;
3626 }
3627
3628 /**
3629 * Return the additional aperture space required by the tree of buffer objects
3630 * rooted at bo.
3631 */
3632 static int
mos_gem_bo_get_aperture_space(struct mos_linux_bo * bo)3633 mos_gem_bo_get_aperture_space(struct mos_linux_bo *bo)
3634 {
3635 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3636 int i;
3637 int total = 0;
3638
3639 if (bo == nullptr || bo_gem->included_in_check_aperture)
3640 return 0;
3641
3642 total += bo->size;
3643 bo_gem->included_in_check_aperture = true;
3644
3645 for (i = 0; i < bo_gem->reloc_count; i++)
3646 total +=
3647 mos_gem_bo_get_aperture_space(bo_gem->
3648 reloc_target_info[i].bo);
3649
3650 return total;
3651 }
3652
3653 /**
3654 * Count the number of buffers in this list that need a fence reg
3655 *
3656 * If the count is greater than the number of available regs, we'll have
3657 * to ask the caller to resubmit a batch with fewer tiled buffers.
3658 *
3659 * This function over-counts if the same buffer is used multiple times.
3660 */
3661 static unsigned int
mos_gem_total_fences(struct mos_linux_bo ** bo_array,int count)3662 mos_gem_total_fences(struct mos_linux_bo ** bo_array, int count)
3663 {
3664 int i;
3665 unsigned int total = 0;
3666
3667 for (i = 0; i < count; i++) {
3668 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
3669
3670 if (bo_gem == nullptr)
3671 continue;
3672
3673 total += bo_gem->reloc_tree_fences;
3674 }
3675 return total;
3676 }
3677
3678 /**
3679 * Clear the flag set by drm_intel_gem_bo_get_aperture_space() so we're ready
3680 * for the next drm_intel_bufmgr_check_aperture_space() call.
3681 */
3682 static void
mos_gem_bo_clear_aperture_space_flag(struct mos_linux_bo * bo)3683 mos_gem_bo_clear_aperture_space_flag(struct mos_linux_bo *bo)
3684 {
3685 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3686 int i;
3687
3688 if (bo == nullptr || !bo_gem->included_in_check_aperture)
3689 return;
3690
3691 bo_gem->included_in_check_aperture = false;
3692
3693 for (i = 0; i < bo_gem->reloc_count; i++)
3694 mos_gem_bo_clear_aperture_space_flag(bo_gem->
3695 reloc_target_info[i].bo);
3696 }
3697
3698 /**
3699 * Return a conservative estimate for the amount of aperture required
3700 * for a collection of buffers. This may double-count some buffers.
3701 */
3702 static unsigned int
mos_gem_estimate_batch_space(struct mos_linux_bo ** bo_array,int count)3703 mos_gem_estimate_batch_space(struct mos_linux_bo **bo_array, int count)
3704 {
3705 int i;
3706 unsigned int total = 0;
3707
3708 for (i = 0; i < count; i++) {
3709 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo_array[i];
3710 if (bo_gem != nullptr)
3711 total += bo_gem->reloc_tree_size;
3712 }
3713 return total;
3714 }
3715
3716 /**
3717 * Return the amount of aperture needed for a collection of buffers.
3718 * This avoids double counting any buffers, at the cost of looking
3719 * at every buffer in the set.
3720 */
3721 static unsigned int
mos_gem_compute_batch_space(struct mos_linux_bo ** bo_array,int count)3722 mos_gem_compute_batch_space(struct mos_linux_bo **bo_array, int count)
3723 {
3724 int i;
3725 unsigned int total = 0;
3726
3727 for (i = 0; i < count; i++) {
3728 total += mos_gem_bo_get_aperture_space(bo_array[i]);
3729 /* For the first buffer object in the array, we get an
3730 * accurate count back for its reloc_tree size (since nothing
3731 * had been flagged as being counted yet). We can save that
3732 * value out as a more conservative reloc_tree_size that
3733 * avoids double-counting target buffers. Since the first
3734 * buffer happens to usually be the batch buffer in our
3735 * callers, this can pull us back from doing the tree
3736 * walk on every new batch emit.
3737 */
3738 if (i == 0) {
3739 struct mos_bo_gem *bo_gem =
3740 (struct mos_bo_gem *) bo_array[i];
3741 bo_gem->reloc_tree_size = total;
3742 }
3743 }
3744
3745 for (i = 0; i < count; i++)
3746 mos_gem_bo_clear_aperture_space_flag(bo_array[i]);
3747 return total;
3748 }
3749
3750 /**
3751 * Return -1 if the batchbuffer should be flushed before attempting to
3752 * emit rendering referencing the buffers pointed to by bo_array.
3753 *
3754 * This is required because if we try to emit a batchbuffer with relocations
3755 * to a tree of buffers that won't simultaneously fit in the aperture,
3756 * the rendering will return an error at a point where the software is not
3757 * prepared to recover from it.
3758 *
3759 * However, we also want to emit the batchbuffer significantly before we reach
3760 * the limit, as a series of batchbuffers each of which references buffers
3761 * covering almost all of the aperture means that at each emit we end up
3762 * waiting to evict a buffer from the last rendering, and we get synchronous
3763 * performance. By emitting smaller batchbuffers, we eat some CPU overhead to
3764 * get better parallelism.
3765 */
3766 static int
mos_gem_check_aperture_space(struct mos_linux_bo ** bo_array,int count)3767 mos_gem_check_aperture_space(struct mos_linux_bo **bo_array, int count)
3768 {
3769 struct mos_bufmgr_gem *bufmgr_gem =
3770 (struct mos_bufmgr_gem *) bo_array[0]->bufmgr;
3771 unsigned int total = 0;
3772 unsigned int threshold = bufmgr_gem->gtt_size * 3 / 4;
3773 int total_fences;
3774
3775 /* Check for fence reg constraints if necessary */
3776 if (bufmgr_gem->available_fences) {
3777 total_fences = mos_gem_total_fences(bo_array, count);
3778 if (total_fences > bufmgr_gem->available_fences)
3779 return -ENOSPC;
3780 }
3781
3782 total = mos_gem_estimate_batch_space(bo_array, count);
3783
3784 if (total > threshold)
3785 total = mos_gem_compute_batch_space(bo_array, count);
3786
3787 if (total > threshold) {
3788 MOS_DBG("check_space: overflowed available aperture, "
3789 "%dkb vs %dkb\n",
3790 total / 1024, (int)bufmgr_gem->gtt_size / 1024);
3791 return -ENOSPC;
3792 } else {
3793 MOS_DBG("drm_check_space: total %dkb vs bufgr %dkb\n", total / 1024,
3794 (int)bufmgr_gem->gtt_size / 1024);
3795 return 0;
3796 }
3797 }
3798
3799 /*
3800 * Disable buffer reuse for objects which are shared with the kernel
3801 * as scanout buffers
3802 */
3803 static int
mos_gem_bo_disable_reuse(struct mos_linux_bo * bo)3804 mos_gem_bo_disable_reuse(struct mos_linux_bo *bo)
3805 {
3806 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3807
3808 bo_gem->reusable = false;
3809 return 0;
3810 }
3811
3812 static int
mos_gem_bo_is_reusable(struct mos_linux_bo * bo)3813 mos_gem_bo_is_reusable(struct mos_linux_bo *bo)
3814 {
3815 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3816
3817 return bo_gem->reusable;
3818 }
3819
3820 static int
_mos_gem_bo_references(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo)3821 _mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3822 {
3823 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
3824 int i;
3825
3826 for (i = 0; i < bo_gem->reloc_count; i++) {
3827 if (bo_gem->reloc_target_info[i].bo == target_bo)
3828 return 1;
3829 if (bo == bo_gem->reloc_target_info[i].bo)
3830 continue;
3831 if (_mos_gem_bo_references(bo_gem->reloc_target_info[i].bo,
3832 target_bo))
3833 return 1;
3834 }
3835
3836 for (i = 0; i< bo_gem->softpin_target_count; i++) {
3837 if (bo_gem->softpin_target[i].bo == target_bo)
3838 return 1;
3839 if (_mos_gem_bo_references(bo_gem->softpin_target[i].bo, target_bo))
3840 return 1;
3841 }
3842
3843 return 0;
3844 }
3845
3846 /** Return true if target_bo is referenced by bo's relocation tree. */
3847 static int
mos_gem_bo_references(struct mos_linux_bo * bo,struct mos_linux_bo * target_bo)3848 mos_gem_bo_references(struct mos_linux_bo *bo, struct mos_linux_bo *target_bo)
3849 {
3850 struct mos_bo_gem *target_bo_gem = (struct mos_bo_gem *) target_bo;
3851
3852 if (bo == nullptr || target_bo == nullptr)
3853 return 0;
3854 if (target_bo_gem->used_as_reloc_target)
3855 return _mos_gem_bo_references(bo, target_bo);
3856 return 0;
3857 }
3858
3859 static void
add_bucket(struct mos_bufmgr_gem * bufmgr_gem,int size)3860 add_bucket(struct mos_bufmgr_gem *bufmgr_gem, int size)
3861 {
3862 unsigned int i = bufmgr_gem->num_buckets;
3863
3864 assert(i < ARRAY_SIZE(bufmgr_gem->cache_bucket));
3865
3866 DRMINITLISTHEAD(&bufmgr_gem->cache_bucket[i].head);
3867 bufmgr_gem->cache_bucket[i].size = size;
3868 bufmgr_gem->num_buckets++;
3869 }
3870
3871 static void
init_cache_buckets(struct mos_bufmgr_gem * bufmgr_gem)3872 init_cache_buckets(struct mos_bufmgr_gem *bufmgr_gem)
3873 {
3874 unsigned long size, cache_max_size = 64 * 1024 * 1024;
3875
3876 /* OK, so power of two buckets was too wasteful of memory.
3877 * Give 3 other sizes between each power of two, to hopefully
3878 * cover things accurately enough. (The alternative is
3879 * probably to just go for exact matching of sizes, and assume
3880 * that for things like composited window resize the tiled
3881 * width/height alignment and rounding of sizes to pages will
3882 * get us useful cache hit rates anyway)
3883 */
3884 add_bucket(bufmgr_gem, 4096);
3885 add_bucket(bufmgr_gem, 4096 * 2);
3886 add_bucket(bufmgr_gem, 4096 * 3);
3887
3888 /* Initialize the linked lists for BO reuse cache. */
3889 for (size = 4 * 4096; size <= cache_max_size; size *= 2) {
3890 add_bucket(bufmgr_gem, size);
3891
3892 add_bucket(bufmgr_gem, size + size * 1 / 4);
3893 add_bucket(bufmgr_gem, size + size * 2 / 4);
3894 add_bucket(bufmgr_gem, size + size * 3 / 4);
3895 }
3896 }
3897
3898 static void
mos_gem_realloc_cache(struct mos_bufmgr * bufmgr,uint8_t alloc_mode)3899 mos_gem_realloc_cache(struct mos_bufmgr *bufmgr, uint8_t alloc_mode)
3900 {
3901 unsigned long size, cache_max_size = 64 * 1024 * 1024, unit_size;
3902 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
3903
3904 // Clean up the pre-allocated cache before re-allocating according
3905 // to alloc_mode
3906 mos_bufmgr_cleanup_cache(bufmgr_gem);
3907
3908 /* OK, so power of two buckets was too wasteful of memory.
3909 * Give 3 other sizes between each power of two, to hopefully
3910 * cover things accurately enough. (The alternative is
3911 * probably to just go for exact matching of sizes, and assume
3912 * that for things like composited window resize the tiled
3913 * width/height alignment and rounding of sizes to pages will
3914 * get us useful cache hit rates anyway)
3915 */
3916 /* alloc_mode 0 is default alloc_mode
3917 * alloc_mode 1 rounding up to 64K for all < 1M
3918 * alloc_mode 2 rounding up to 2M for size> 1M
3919 * alloc_mode 3 rounding up to 2M for size > 1M and 64K for size <= 1M */
3920 if( alloc_mode > 3 )
3921 alloc_mode = 0;
3922
3923 if ( 0 == alloc_mode || 2 == alloc_mode)
3924 {
3925 // < 1M normal alloc_mode
3926 add_bucket(bufmgr_gem, 4096);
3927 add_bucket(bufmgr_gem, 4096 * 2);
3928 add_bucket(bufmgr_gem, 4096 * 3);
3929 /* Initialize the linked lists for BO reuse cache. */
3930 for (size = 4 * 4096; size < 1024 * 1024; size *= 2) {
3931 add_bucket(bufmgr_gem, size);
3932 add_bucket(bufmgr_gem, size + size * 1 / 4);
3933 add_bucket(bufmgr_gem, size + size * 2 / 4);
3934 add_bucket(bufmgr_gem, size + size * 3 / 4);
3935 }
3936
3937 add_bucket(bufmgr_gem, 1024 * 1024);
3938 }
3939 if (1 == alloc_mode || 3 == alloc_mode)
3940 {
3941 // < 1M 64k alignment
3942 unit_size = 64 * 1024;
3943 for (size = unit_size; size <= 1024 * 1024; size += unit_size)
3944 {
3945 add_bucket(bufmgr_gem, size);
3946 }
3947 }
3948 if( 0 == alloc_mode || 1 == alloc_mode)
3949 {
3950 //> 1M is normal alloc_mode
3951 add_bucket(bufmgr_gem, 1280 * 1024);
3952 add_bucket(bufmgr_gem, 1536 * 1024);
3953 add_bucket(bufmgr_gem, 1792 * 1024);
3954
3955 for (size = 2 * 1024 * 1024; size < cache_max_size; size *= 2) {
3956 add_bucket(bufmgr_gem, size);
3957 add_bucket(bufmgr_gem, size + size * 1 / 4);
3958 add_bucket(bufmgr_gem, size + size * 2 / 4);
3959 add_bucket(bufmgr_gem, size + size * 3 / 4);
3960 }
3961 }
3962 if( 2 == alloc_mode || 3 == alloc_mode)
3963 {
3964 //> 1M rolling to 2M
3965 unit_size = 2 * 1024 * 1024;
3966 add_bucket(bufmgr_gem, unit_size);
3967 add_bucket(bufmgr_gem, 3 * 1024 * 1024);
3968
3969 for (size = 4 * 1024 * 1024; size <= cache_max_size; size += unit_size)
3970 {
3971 add_bucket(bufmgr_gem, size);
3972 }
3973 }
3974 }
3975
3976 /**
3977 * Get the PCI ID for the device. This can be overridden by setting the
3978 * INTEL_DEVID_OVERRIDE environment variable to the desired ID.
3979 */
3980 static int
get_pci_device_id(struct mos_bufmgr_gem * bufmgr_gem)3981 get_pci_device_id(struct mos_bufmgr_gem *bufmgr_gem)
3982 {
3983 char *devid_override;
3984 int devid = 0;
3985 int ret;
3986 drm_i915_getparam_t gp;
3987
3988 if (geteuid() == getuid()) {
3989 devid_override = getenv("INTEL_DEVID_OVERRIDE");
3990 if (devid_override) {
3991 bufmgr_gem->no_exec = true;
3992 return strtod(devid_override, nullptr);
3993 }
3994 }
3995
3996 memclear(gp);
3997 gp.param = I915_PARAM_CHIPSET_ID;
3998 gp.value = &devid;
3999 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
4000 if (ret) {
4001 fprintf(stderr, "get chip id failed: %d [%d]\n", ret, errno);
4002 fprintf(stderr, "param: %d, val: %d\n", gp.param, *gp.value);
4003 }
4004 return devid;
4005 }
4006
4007 static int
mos_gem_get_devid(struct mos_bufmgr * bufmgr)4008 mos_gem_get_devid(struct mos_bufmgr *bufmgr)
4009 {
4010 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4011
4012 return bufmgr_gem->pci_device;
4013 }
4014
mos_gem_ctx_set_user_ctx_params(struct mos_linux_context * context)4015 int mos_gem_ctx_set_user_ctx_params(struct mos_linux_context *context)
4016 {
4017 /*
4018 * INTEL_I915_CTX_CONTROL=0, 1, 2, 3
4019 * 0: default, do nothing
4020 * 1: disable ctx recoverable
4021 * 2: disable ctx bannable
4022 * 3: disable both
4023 * */
4024
4025 if (context == nullptr)
4026 {
4027 return -EINVAL;
4028 }
4029
4030 int ret = 0;
4031 char *user_ctx_env = getenv("INTEL_I915_CTX_CONTROL");
4032 bool disable_recoverable = false;
4033 bool disable_bannable = false;
4034 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)context->bufmgr;
4035
4036 if (user_ctx_env != nullptr)
4037 {
4038 uint8_t user_ctx_env_value = (uint8_t)atoi(user_ctx_env);
4039 if(user_ctx_env_value > 3)
4040 {
4041 MOS_DBG("INTEL_I915_CTX_CONTROL: invalid value %u setting\n",
4042 user_ctx_env_value);
4043 }
4044 else
4045 {
4046 if (user_ctx_env_value & 0x1)
4047 {
4048 disable_recoverable = true;
4049 }
4050
4051 if(user_ctx_env_value & 0x2)
4052 {
4053 disable_bannable = true;
4054 }
4055 }
4056 }
4057
4058 if(disable_recoverable)
4059 {
4060 ret = mos_set_context_param(context,
4061 0,
4062 I915_CONTEXT_PARAM_RECOVERABLE,
4063 0);
4064 if(ret != 0) {
4065 MOS_DBG("I915_CONTEXT_PARAM_RECOVERABLE failed: %s\n",
4066 strerror(errno));
4067 }
4068 else
4069 {
4070 MOS_DBG("successfull to disable context recoverable\n");
4071 }
4072 }
4073
4074 if(disable_bannable)
4075 {
4076 ret = mos_set_context_param(context,
4077 0,
4078 I915_CONTEXT_PARAM_BANNABLE,
4079 0);
4080 if(ret != 0) {
4081 MOS_DBG("I915_CONTEXT_PARAM_BANNABLE failed: %s\n",
4082 strerror(errno));
4083 }
4084 else
4085 {
4086 MOS_DBG("successfull to disable context bannable\n");
4087 }
4088 }
4089
4090 return ret;
4091 }
4092
4093 static struct mos_linux_context *
mos_gem_context_create(struct mos_bufmgr * bufmgr)4094 mos_gem_context_create(struct mos_bufmgr *bufmgr)
4095 {
4096 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4097 struct drm_i915_gem_context_create create;
4098 struct mos_linux_context *context = nullptr;
4099 int ret;
4100
4101 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4102 if (!context)
4103 return nullptr;
4104
4105 memclear(create);
4106 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
4107 if (ret != 0) {
4108 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4109 strerror(errno));
4110 free(context);
4111 return nullptr;
4112 }
4113
4114 context->ctx_id = create.ctx_id;
4115 context->bufmgr = bufmgr;
4116
4117 ret = mos_gem_ctx_set_user_ctx_params(context);
4118
4119 return context;
4120 }
4121
4122 static void
mos_gem_context_destroy(struct mos_linux_context * ctx)4123 mos_gem_context_destroy(struct mos_linux_context *ctx)
4124 {
4125 struct mos_bufmgr_gem *bufmgr_gem;
4126 struct drm_i915_gem_context_destroy destroy;
4127 int ret;
4128
4129 if (ctx == nullptr)
4130 return;
4131
4132 memclear(destroy);
4133
4134 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4135 destroy.ctx_id = ctx->ctx_id;
4136 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY,
4137 &destroy);
4138 if (ret != 0)
4139 fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
4140 strerror(errno));
4141
4142 free(ctx);
4143 }
4144
4145 static int
mos_bufmg_get_reset_stats(struct mos_linux_context * ctx,uint32_t * reset_count,uint32_t * active,uint32_t * pending)4146 mos_bufmg_get_reset_stats(struct mos_linux_context *ctx,
4147 uint32_t *reset_count,
4148 uint32_t *active,
4149 uint32_t *pending)
4150 {
4151 struct mos_bufmgr_gem *bufmgr_gem;
4152 struct drm_i915_reset_stats stats;
4153 int ret;
4154
4155 if (ctx == nullptr)
4156 return -EINVAL;
4157
4158 memclear(stats);
4159
4160 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4161 stats.ctx_id = ctx->ctx_id;
4162 ret = drmIoctl(bufmgr_gem->fd,
4163 DRM_IOCTL_I915_GET_RESET_STATS,
4164 &stats);
4165 if (ret == 0) {
4166 if (reset_count != nullptr)
4167 *reset_count = stats.reset_count;
4168
4169 if (active != nullptr)
4170 *active = stats.batch_active;
4171
4172 if (pending != nullptr)
4173 *pending = stats.batch_pending;
4174 }
4175
4176 return ret;
4177 }
4178
mos_bufmgr_hweight8(struct mos_linux_context * ctx,uint8_t w)4179 static unsigned int mos_bufmgr_hweight8(struct mos_linux_context *ctx, uint8_t w)
4180 {
4181 uint32_t i, weight = 0;
4182
4183 for (i=0; i<8; i++)
4184 {
4185 weight += !!((w) & (1UL << i));
4186 }
4187 return weight;
4188 }
4189
mos_bufmgr_switch_off_n_bits(struct mos_linux_context * ctx,uint8_t in_mask,int n)4190 static uint8_t mos_bufmgr_switch_off_n_bits(struct mos_linux_context *ctx, uint8_t in_mask, int n)
4191 {
4192 int i,count;
4193 uint8_t bi,out_mask;
4194
4195 assert (n>0 && n<=8);
4196
4197 out_mask = in_mask;
4198 count = n;
4199 for(i=0; i<8; i++)
4200 {
4201 bi = 1UL<<i;
4202 if (bi & in_mask)
4203 {
4204 out_mask &= ~bi;
4205 count--;
4206 }
4207 if (count==0)
4208 {
4209 break;
4210 }
4211 }
4212 return out_mask;
4213 }
4214
4215 static int
mos_bufmgr_get_context_param_sseu(struct mos_linux_context * ctx,struct drm_i915_gem_context_param_sseu * sseu)4216 mos_bufmgr_get_context_param_sseu(struct mos_linux_context *ctx,
4217 struct drm_i915_gem_context_param_sseu *sseu)
4218 {
4219 struct mos_bufmgr_gem *bufmgr_gem;
4220 struct drm_i915_gem_context_param context_param;
4221 int ret;
4222
4223 if (ctx == nullptr)
4224 return -EINVAL;
4225
4226 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4227 memset(&context_param, 0, sizeof(context_param));
4228 context_param.ctx_id = ctx->ctx_id;
4229 context_param.param = I915_CONTEXT_PARAM_SSEU;
4230 context_param.value = (uint64_t) sseu;
4231 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
4232
4233 ret = drmIoctl(bufmgr_gem->fd,
4234 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
4235 &context_param);
4236
4237 return ret;
4238 }
4239
4240 static int
mos_bufmgr_set_context_param_sseu(struct mos_linux_context * ctx,struct drm_i915_gem_context_param_sseu sseu)4241 mos_bufmgr_set_context_param_sseu(struct mos_linux_context *ctx,
4242 struct drm_i915_gem_context_param_sseu sseu)
4243 {
4244 struct mos_bufmgr_gem *bufmgr_gem;
4245 struct drm_i915_gem_context_param context_param;
4246 int ret;
4247
4248 if (ctx == nullptr)
4249 return -EINVAL;
4250
4251 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4252 memset(&context_param, 0, sizeof(context_param));
4253 context_param.ctx_id = ctx->ctx_id;
4254 context_param.param = I915_CONTEXT_PARAM_SSEU;
4255 context_param.value = (uint64_t) &sseu;
4256 context_param.size = sizeof(struct drm_i915_gem_context_param_sseu);
4257
4258 ret = drmIoctl(bufmgr_gem->fd,
4259 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
4260 &context_param);
4261
4262 return ret;
4263 }
4264
4265 static int
mos_gem_get_context_param(struct mos_linux_context * ctx,uint32_t size,uint64_t param,uint64_t * value)4266 mos_gem_get_context_param(struct mos_linux_context *ctx,
4267 uint32_t size,
4268 uint64_t param,
4269 uint64_t *value)
4270 {
4271 struct mos_bufmgr_gem *bufmgr_gem;
4272 struct drm_i915_gem_context_param context_param;
4273 int ret;
4274
4275 if (ctx == nullptr)
4276 return -EINVAL;
4277
4278 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4279 context_param.ctx_id = ctx->ctx_id;
4280 context_param.size = size;
4281 context_param.param = param;
4282 context_param.value = 0;
4283
4284 ret = drmIoctl(bufmgr_gem->fd,
4285 DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM,
4286 &context_param);
4287 *value = context_param.value;
4288
4289 return ret;
4290 }
4291
4292 static int
mos_gem_set_context_param(struct mos_linux_context * ctx,uint32_t size,uint64_t param,uint64_t value)4293 mos_gem_set_context_param(struct mos_linux_context *ctx,
4294 uint32_t size,
4295 uint64_t param,
4296 uint64_t value)
4297 {
4298 struct mos_bufmgr_gem *bufmgr_gem;
4299 struct drm_i915_gem_context_param context_param;
4300 int ret;
4301
4302 if (ctx == nullptr)
4303 return -EINVAL;
4304
4305 bufmgr_gem = (struct mos_bufmgr_gem *)ctx->bufmgr;
4306 context_param.ctx_id = ctx->ctx_id;
4307 context_param.size = size;
4308 context_param.param = param;
4309 context_param.value = value;
4310
4311 ret = drmIoctl(bufmgr_gem->fd,
4312 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM,
4313 &context_param);
4314
4315 return ret;
4316 }
4317
4318 static int
mos_bufmg_reg_read(struct mos_bufmgr * bufmgr,uint32_t offset,uint64_t * result)4319 mos_bufmg_reg_read(struct mos_bufmgr *bufmgr,
4320 uint32_t offset,
4321 uint64_t *result)
4322 {
4323 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4324 struct drm_i915_reg_read reg_read;
4325 int ret;
4326
4327 memclear(reg_read);
4328 reg_read.offset = offset;
4329
4330 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_REG_READ, ®_read);
4331
4332 *result = reg_read.val;
4333 return ret;
4334 }
4335
4336 static pthread_mutex_t bufmgr_list_mutex = PTHREAD_MUTEX_INITIALIZER;
4337 static drmMMListHead bufmgr_list = { &bufmgr_list, &bufmgr_list };
4338
4339 static struct mos_bufmgr_gem *
mos_bufmgr_gem_find(int fd)4340 mos_bufmgr_gem_find(int fd)
4341 {
4342 struct mos_bufmgr_gem *bufmgr_gem;
4343
4344 DRMLISTFOREACHENTRY(bufmgr_gem, &bufmgr_list, managers) {
4345 if (bufmgr_gem->fd == fd) {
4346 atomic_inc(&bufmgr_gem->refcount);
4347 return bufmgr_gem;
4348 }
4349 }
4350
4351 return nullptr;
4352 }
4353
4354 static void
mos_bufmgr_gem_unref(struct mos_bufmgr * bufmgr)4355 mos_bufmgr_gem_unref(struct mos_bufmgr *bufmgr)
4356 {
4357 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4358
4359 if (atomic_add_unless(&bufmgr_gem->refcount, -1, 1)) {
4360 pthread_mutex_lock(&bufmgr_list_mutex);
4361
4362 if (atomic_dec_and_test(&bufmgr_gem->refcount)) {
4363 DRMLISTDEL(&bufmgr_gem->managers);
4364 mos_bufmgr_gem_destroy(bufmgr);
4365 }
4366
4367 pthread_mutex_unlock(&bufmgr_list_mutex);
4368 }
4369 }
4370
4371 static int
mos_gem_get_memory_info(struct mos_bufmgr * bufmgr,char * info,uint32_t length)4372 mos_gem_get_memory_info(struct mos_bufmgr *bufmgr, char *info, uint32_t length)
4373 {
4374 return 0;
4375 }
4376
mos_gem_enable_softpin(struct mos_bufmgr * bufmgr,bool va1m_align)4377 static void mos_gem_enable_softpin(struct mos_bufmgr *bufmgr, bool va1m_align)
4378 {
4379 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4380 bufmgr_gem->use_softpin = true;
4381 bufmgr_gem->softpin_va1Malign = va1m_align;
4382 }
4383
mos_gem_enable_vmbind(struct mos_bufmgr * bufmgr)4384 static void mos_gem_enable_vmbind(struct mos_bufmgr *bufmgr)
4385 {
4386 }
4387
mos_gem_disable_object_capture(struct mos_bufmgr * bufmgr)4388 static void mos_gem_disable_object_capture(struct mos_bufmgr *bufmgr)
4389 {
4390 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4391 if (bufmgr_gem != nullptr)
4392 {
4393 bufmgr_gem->object_capture_disabled = true;
4394 }
4395 }
4396
4397 static struct mos_linux_context *
mos_gem_context_create_ext(struct mos_bufmgr * bufmgr,__u32 flags,bool bContextProtected)4398 mos_gem_context_create_ext(struct mos_bufmgr *bufmgr, __u32 flags, bool bContextProtected)
4399 {
4400 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4401 struct drm_i915_gem_context_create_ext create;
4402 struct mos_linux_context *context = nullptr;
4403 struct drm_i915_gem_context_create_ext_setparam p_protected;
4404 struct drm_i915_gem_context_create_ext_setparam p_norecover;
4405 int ret;
4406
4407 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4408 if (!context)
4409 return nullptr;
4410
4411 memclear(create);
4412 create.flags = flags;
4413 create.extensions = 0;
4414
4415 if (bContextProtected)
4416 {
4417 memclear(p_protected);
4418 memclear(p_norecover);
4419 p_protected.base.next_extension = 0;
4420 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4421 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
4422 p_protected.param.value = 1;
4423
4424 p_norecover.base.next_extension = (uintptr_t)&p_protected;
4425 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4426 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
4427 p_norecover.param.value = 0;
4428
4429 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
4430 create.extensions = (uintptr_t)&p_norecover;
4431 }
4432 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
4433 if (ret != 0) {
4434 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4435 strerror(errno));
4436 free(context);
4437 return nullptr;
4438 }
4439
4440 context->ctx_id = create.ctx_id;
4441 context->bufmgr = bufmgr;
4442
4443 ret = mos_gem_ctx_set_user_ctx_params(context);
4444
4445 return context;
4446 }
4447
mos_gem_vm_create(struct mos_bufmgr * bufmgr)4448 static __u32 mos_gem_vm_create(struct mos_bufmgr *bufmgr)
4449 {
4450 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4451 struct drm_i915_gem_vm_control *vm = nullptr;
4452 __u32 vm_id;
4453 int ret;
4454
4455 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
4456 if (nullptr == vm)
4457 {
4458 MOS_DBG("vm calloc failed\n" );
4459 return INVALID_VM;
4460 }
4461
4462 memset(vm, 0, sizeof(*vm));
4463
4464 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_CREATE, vm);
4465 if (ret != 0) {
4466 MOS_DBG("DRM_IOCTL_I915_GEM_VM_CREATE failed: %s\n",
4467 strerror(errno));
4468 free(vm);
4469 return INVALID_VM;
4470 }
4471
4472 vm_id = vm->vm_id;
4473 free(vm);
4474
4475 return vm_id;
4476 }
4477
mos_gem_vm_destroy(struct mos_bufmgr * bufmgr,__u32 vm_id)4478 static void mos_gem_vm_destroy(struct mos_bufmgr *bufmgr, __u32 vm_id)
4479 {
4480 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4481 struct drm_i915_gem_vm_control *vm = nullptr;
4482 int ret;
4483
4484 if (vm_id == INVALID_VM)
4485 {
4486 MOS_DBG("input invalid param\n" );
4487 return;
4488 }
4489
4490 vm = (struct drm_i915_gem_vm_control *)calloc(1, sizeof(struct drm_i915_gem_vm_control));
4491
4492 if (nullptr == vm)
4493 {
4494 MOS_DBG("vm calloc failed\n" );
4495 return;
4496 }
4497
4498 vm->vm_id = vm_id;
4499 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_VM_DESTROY, vm);
4500 if (ret != 0) {
4501 MOS_DBG("DRM_IOCTL_I915_GEM_VM_DESTROY failed: %s\n",
4502 strerror(errno));
4503 }
4504
4505 free(vm);
4506 }
4507
4508 static struct mos_linux_context *
mos_gem_context_create_shared(struct mos_bufmgr * bufmgr,mos_linux_context * ctx,__u32 flags,bool bContextProtected,void * engine_map,uint8_t ctx_width,uint8_t num_placements,uint32_t ctx_type)4509 mos_gem_context_create_shared(struct mos_bufmgr *bufmgr,
4510 mos_linux_context* ctx,
4511 __u32 flags,
4512 bool bContextProtected,
4513 void *engine_map,
4514 uint8_t ctx_width,
4515 uint8_t num_placements,
4516 uint32_t ctx_type)
4517 {
4518 MOS_UNUSED(engine_map);
4519 MOS_UNUSED(ctx_width);
4520 MOS_UNUSED(num_placements);
4521 MOS_UNUSED(ctx_type);
4522
4523 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4524 struct drm_i915_gem_context_create_ext create;
4525 struct mos_linux_context *context = nullptr;
4526 struct drm_i915_gem_context_create_ext_setparam p_protected;
4527 struct drm_i915_gem_context_create_ext_setparam p_norecover;
4528 int ret;
4529
4530 if (ctx == nullptr || ctx->vm_id == INVALID_VM)
4531 return nullptr;
4532
4533 context = (struct mos_linux_context *)calloc(1, sizeof(*context));
4534 if (!context)
4535 return nullptr;
4536
4537 memclear(create);
4538 create.flags = flags;
4539 create.extensions = 0;
4540 if (bContextProtected)
4541 {
4542 memclear(p_protected);
4543 memclear(p_norecover);
4544 p_protected.base.next_extension = 0;
4545 p_protected.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4546 p_protected.param.param = I915_CONTEXT_PARAM_PROTECTED_CONTENT;
4547 p_protected.param.value = 1;
4548
4549 p_norecover.base.next_extension = (uintptr_t)&p_protected;
4550 p_norecover.base.name = I915_CONTEXT_CREATE_EXT_SETPARAM;
4551 p_norecover.param.param = I915_CONTEXT_PARAM_RECOVERABLE;
4552 p_norecover.param.value = 0;
4553
4554 create.flags = flags|I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS;
4555 create.extensions = (uintptr_t)&p_norecover;
4556 }
4557 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE_EXT, &create);
4558 if (ret != 0) {
4559 MOS_DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n",
4560 strerror(errno));
4561 free(context);
4562 return nullptr;
4563 }
4564
4565 context->ctx_id = create.ctx_id;
4566 context->bufmgr = bufmgr;
4567
4568 ret = mos_set_context_param(context,
4569 0,
4570 I915_CONTEXT_PARAM_VM,
4571 ctx->vm_id);
4572 if(ret != 0) {
4573 MOS_DBG("I915_CONTEXT_PARAM_VM failed: %s\n",
4574 strerror(errno));
4575 free(context);
4576 return nullptr;
4577 }
4578
4579 ret = mos_gem_ctx_set_user_ctx_params(context);
4580
4581 return context;
4582 }
4583
mos_bufmgr_query_engines_count(struct mos_bufmgr * bufmgr,unsigned int * nengine)4584 static int mos_bufmgr_query_engines_count(struct mos_bufmgr *bufmgr,
4585 unsigned int *nengine)
4586 {
4587 assert(bufmgr);
4588 assert(nengine);
4589 int fd = ((struct mos_bufmgr_gem*)bufmgr)->fd;
4590 struct drm_i915_query query;
4591 struct drm_i915_query_item query_item;
4592 struct drm_i915_query_engine_info *engines = nullptr;
4593 int ret, len;
4594
4595 memclear(query_item);
4596 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4597 query_item.length = 0;
4598 memclear(query);
4599 query.num_items = 1;
4600 query.items_ptr = (uintptr_t)&query_item;
4601
4602 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4603 if (ret || query_item.length == 0)
4604 {
4605 *nengine = 0;
4606 return ret;
4607 }
4608
4609 len = query_item.length;
4610
4611 engines = (drm_i915_query_engine_info *)malloc(len);
4612 if (nullptr == engines)
4613 {
4614 *nengine = 0;
4615 ret = -ENOMEM;
4616 return ret;
4617 }
4618
4619 memset(engines, 0, len);
4620 memclear(query_item);
4621 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4622 query_item.length = len;
4623 query_item.data_ptr = (uintptr_t)engines;
4624 memclear(query);
4625 query.num_items = 1;
4626 query.items_ptr = (uintptr_t)&query_item;
4627
4628 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4629
4630 *nengine = ret ? 0 : engines->num_engines;
4631
4632 if(engines)
4633 {
4634 free(engines);
4635 }
4636 return ret;
4637 }
4638
mos_bufmgr_query_engines(struct mos_bufmgr * bufmgr,__u16 engine_class,__u64 caps,unsigned int * nengine,void * engine_map)4639 static int mos_bufmgr_query_engines(struct mos_bufmgr *bufmgr,
4640 __u16 engine_class,
4641 __u64 caps,
4642 unsigned int *nengine,
4643 void *engine_map)
4644 {
4645 struct i915_engine_class_instance *ci = (struct i915_engine_class_instance *)engine_map;
4646 if((bufmgr == nullptr) || (nengine == nullptr) || (ci == nullptr))
4647 {
4648 return -EINVAL;
4649 }
4650
4651 struct drm_i915_query query;
4652 struct drm_i915_query_item query_item;
4653 struct drm_i915_query_engine_info *engines = nullptr;
4654 int ret, len;
4655 int fd = ((struct mos_bufmgr_gem*)bufmgr)->fd;
4656
4657 memclear(query_item);
4658 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4659 query_item.length = 0;
4660 memclear(query);
4661 query.num_items = 1;
4662 query.items_ptr = (uintptr_t)&query_item;
4663
4664 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4665 if (ret)
4666 {
4667 goto fini;
4668 }
4669
4670 len = query_item.length;
4671 if(len == 0)
4672 {
4673 goto fini;
4674 }
4675
4676 engines = (drm_i915_query_engine_info *)malloc(len);
4677 if (nullptr == engines)
4678 {
4679 ret = -ENOMEM;
4680 goto fini;
4681 }
4682 memset(engines,0,len);
4683 memclear(query_item);
4684 query_item.query_id = DRM_I915_QUERY_ENGINE_INFO;
4685 query_item.length = len;
4686 query_item.data_ptr = (uintptr_t)engines;
4687 memclear(query);
4688 query.num_items = 1;
4689 query.items_ptr = (uintptr_t)&query_item;
4690
4691 ret = drmIoctl(fd, DRM_IOCTL_I915_QUERY, &query);
4692 if (ret)
4693 {
4694 goto fini;
4695 }
4696
4697 int i, num;
4698 for (i = 0, num = 0; i < engines->num_engines; i++) {
4699 struct drm_i915_engine_info *engine =
4700 (struct drm_i915_engine_info *)&engines->engines[i];
4701 if ( engine_class == engine->engine.engine_class
4702 && ((caps & engine->capabilities) == caps ))
4703 {
4704 ci->engine_class = engine_class;
4705 ci->engine_instance = engine->engine.engine_instance;
4706 ci++;
4707 num++;
4708 }
4709 if (num > *nengine)
4710 {
4711 fprintf(stderr,"%s: Number of engine instances out of range, %d,%d\n",
4712 __FUNCTION__, num, *nengine);
4713 goto fini;
4714 }
4715 }
4716 *nengine = num;
4717
4718
4719 fini:
4720 if (engines)
4721 free(engines);
4722 return ret;
4723 }
4724
mos_bufmgr_get_engine_class_size()4725 static size_t mos_bufmgr_get_engine_class_size()
4726 {
4727 return sizeof(struct i915_engine_class_instance);
4728 }
4729
4730 static int
mos_bufmgr_query_sys_engines(struct mos_bufmgr * bufmgr,MEDIA_SYSTEM_INFO * gfx_info)4731 mos_bufmgr_query_sys_engines(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
4732 {
4733 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem *)bufmgr;
4734 int ret;
4735
4736 if (nullptr == gfx_info)
4737 {
4738 return -EINVAL;
4739 }
4740
4741 unsigned int maxNengine = 0;
4742 if((gfx_info->VDBoxInfo.NumberOfVDBoxEnabled == 0)
4743 || (gfx_info->VEBoxInfo.NumberOfVEBoxEnabled == 0))
4744 {
4745 if (mos_query_engines_count(bufmgr, &maxNengine) || (maxNengine == 0))
4746 {
4747 MOS_OS_ASSERTMESSAGE("Failed to query engines count.\n");
4748 return -ENODEV;
4749 }
4750 }
4751
4752 if (gfx_info->VDBoxInfo.NumberOfVDBoxEnabled == 0)
4753 {
4754 unsigned int nengine = maxNengine;
4755 struct i915_engine_class_instance *uengines = nullptr;
4756 uengines = (struct i915_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct i915_engine_class_instance));
4757 if (nullptr == uengines)
4758 {
4759 return -ENOMEM;
4760 }
4761 ret = mos_bufmgr_query_engines(bufmgr, I915_ENGINE_CLASS_VIDEO, 0, &nengine, (void *)uengines);
4762 if (ret)
4763 {
4764 MOS_OS_ASSERTMESSAGE("Failed to query vdbox engine\n");
4765 MOS_SafeFreeMemory(uengines);
4766 return -ENODEV;
4767 }
4768 else
4769 {
4770 gfx_info->VDBoxInfo.NumberOfVDBoxEnabled = nengine;
4771 }
4772
4773 for (int i=0; i<nengine; i++)
4774 {
4775 gfx_info->VDBoxInfo.Instances.VDBoxEnableMask |= 1<<uengines[i].engine_instance;
4776 }
4777
4778 MOS_SafeFreeMemory(uengines);
4779 }
4780
4781 if (gfx_info->VEBoxInfo.NumberOfVEBoxEnabled == 0)
4782 {
4783 unsigned int nengine = maxNengine;
4784 struct i915_engine_class_instance *uengines = nullptr;
4785 uengines = (struct i915_engine_class_instance *)MOS_AllocAndZeroMemory(nengine * sizeof(struct i915_engine_class_instance));
4786 if (nullptr == uengines)
4787 {
4788 return -ENOMEM;
4789 }
4790 ret = mos_bufmgr_query_engines(bufmgr, I915_ENGINE_CLASS_VIDEO_ENHANCE, 0, &nengine, (void *)uengines);
4791 if (ret)
4792 {
4793 MOS_OS_ASSERTMESSAGE("Failed to query vebox engine\n");
4794 MOS_SafeFreeMemory(uengines);
4795 return -ENODEV;
4796 }
4797 else
4798 {
4799 MOS_OS_ASSERT(nengine <= maxNengine);
4800 gfx_info->VEBoxInfo.NumberOfVEBoxEnabled = nengine;
4801 }
4802
4803 MOS_SafeFreeMemory(uengines);
4804 }
4805
4806 return 0;
4807 }
4808
mos_gem_select_fixed_engine(struct mos_bufmgr * bufmgr,void * engine_map,uint32_t * nengine,uint32_t fixed_instance_mask)4809 void mos_gem_select_fixed_engine(struct mos_bufmgr *bufmgr,
4810 void *engine_map,
4811 uint32_t *nengine,
4812 uint32_t fixed_instance_mask)
4813 {
4814 MOS_UNUSED(bufmgr);
4815 #if (DEBUG || _RELEASE_INTERNAL)
4816 if (fixed_instance_mask)
4817 {
4818 struct i915_engine_class_instance *_engine_map = (struct i915_engine_class_instance *)engine_map;
4819 auto unselect_index = 0;
4820 for(auto bit = 0; bit < *nengine; bit++)
4821 {
4822 if(((fixed_instance_mask >> bit) & 0x1) && (bit > unselect_index))
4823 {
4824 _engine_map[unselect_index].engine_class = _engine_map[bit].engine_class;
4825 _engine_map[unselect_index].engine_instance = _engine_map[bit].engine_instance;
4826 _engine_map[bit].engine_class = 0;
4827 _engine_map[bit].engine_instance = 0;
4828 unselect_index++;
4829 }
4830 else if(((fixed_instance_mask >> bit) & 0x1) && (bit == unselect_index))
4831 {
4832 unselect_index++;
4833 }
4834 else if(!((fixed_instance_mask >> bit) & 0x1))
4835 {
4836 _engine_map[bit].engine_class = 0;
4837 _engine_map[bit].engine_instance = 0;
4838 }
4839 }
4840 *nengine = unselect_index;
4841 }
4842 #else
4843 MOS_UNUSED(engine_map);
4844 MOS_UNUSED(nengine);
4845 MOS_UNUSED(fixed_instance_mask);
4846 #endif
4847
4848 }
4849
mos_gem_set_context_param_parallel(struct mos_linux_context * ctx,struct i915_engine_class_instance * ci,unsigned int count)4850 static int mos_gem_set_context_param_parallel(struct mos_linux_context *ctx,
4851 struct i915_engine_class_instance *ci,
4852 unsigned int count)
4853 {
4854 if((ctx == nullptr) || (ci == nullptr) || (count <= 0))
4855 {
4856 return -EINVAL;
4857 }
4858
4859 int ret = 0;
4860 uint32_t size = 0;
4861 int n;
4862 struct i915_context_engines_parallel_submit* parallel_submit = nullptr;
4863 struct i915_context_param_engines* set_engines = nullptr;
4864
4865 size = sizeof(struct i915_context_engines_parallel_submit) + count * sizeof(*ci);
4866 parallel_submit = (struct i915_context_engines_parallel_submit*)malloc(size);
4867 if(parallel_submit == nullptr)
4868 {
4869 ret = -ENOMEM;
4870 goto fini;
4871 }
4872 memset(parallel_submit, 0, size);
4873 parallel_submit->base.name = I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT;
4874 parallel_submit->engine_index = 0;
4875 parallel_submit->width = count;
4876 parallel_submit->num_siblings = 1;
4877 for(int i = 0; i < count; i++)
4878 {
4879 parallel_submit->engines[i] = ci[i];
4880 }
4881
4882 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4883 size = sizeof(struct i915_context_param_engines) + sizeof(struct i915_engine_class_instance);
4884 set_engines = (struct i915_context_param_engines*) malloc(size);
4885 if(set_engines == nullptr)
4886 {
4887 ret = -ENOMEM;
4888 goto fini;
4889 }
4890 set_engines->extensions = (uintptr_t)(parallel_submit);
4891 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4892 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4893
4894 ret = mos_set_context_param(ctx,
4895 size,
4896 I915_CONTEXT_PARAM_ENGINES,
4897 (uintptr_t)set_engines);
4898 fini:
4899 if (set_engines)
4900 free(set_engines);
4901 if (parallel_submit)
4902 free(parallel_submit);
4903 return ret;
4904 }
4905
mos_gem_set_context_param_load_balance(struct mos_linux_context * ctx,struct i915_engine_class_instance * ci,unsigned int count)4906 static int mos_gem_set_context_param_load_balance(struct mos_linux_context *ctx,
4907 struct i915_engine_class_instance *ci,
4908 unsigned int count)
4909 {
4910 int ret;
4911 uint32_t size;
4912 struct i915_context_engines_load_balance* balancer = nullptr;
4913 struct i915_context_param_engines* set_engines = nullptr;
4914
4915 MOS_OS_CHECK_CONDITION(ci == nullptr, "Invalid (nullptr) Pointer.", EINVAL);
4916 MOS_OS_CHECK_CONDITION(count == 0, "Invalid input parameter. Number of engines must be > 0.", EINVAL);
4917
4918 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
4919 size = sizeof(struct i915_context_engines_load_balance) + count * sizeof(*ci);
4920 balancer = (struct i915_context_engines_load_balance*)malloc(size);
4921 if (NULL == balancer)
4922 {
4923 ret = -ENOMEM;
4924 goto fini;
4925 }
4926 memset(balancer, 0, size);
4927 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
4928 balancer->num_siblings = count;
4929 memcpy(balancer->engines, ci, count * sizeof(*ci));
4930
4931 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4932 size = sizeof(uint64_t) + sizeof(*ci);
4933 set_engines = (struct i915_context_param_engines*) malloc(size);
4934 if (NULL == set_engines)
4935 {
4936 ret = -ENOMEM;
4937 goto fini;
4938 }
4939 set_engines->extensions = (uintptr_t)(balancer);
4940 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
4941 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
4942
4943 ret = mos_set_context_param(ctx,
4944 size,
4945 I915_CONTEXT_PARAM_ENGINES,
4946 (uintptr_t)set_engines);
4947 fini:
4948 if (set_engines)
4949 free(set_engines);
4950 if (balancer)
4951 free(balancer);
4952 return ret;
4953 }
4954
mos_gem_set_context_param_bond(struct mos_linux_context * ctx,struct i915_engine_class_instance master_ci,struct i915_engine_class_instance * bond_ci,unsigned int bond_count)4955 static int mos_gem_set_context_param_bond(struct mos_linux_context *ctx,
4956 struct i915_engine_class_instance master_ci,
4957 struct i915_engine_class_instance *bond_ci,
4958 unsigned int bond_count)
4959 {
4960 int ret;
4961 uint32_t size;
4962 struct i915_context_engines_load_balance* balancer = nullptr;
4963 struct i915_context_engines_bond *bond = nullptr;
4964 struct i915_context_param_engines* set_engines = nullptr;
4965
4966 assert(bond_ci);
4967
4968 /* I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE */
4969 size = sizeof(struct i915_context_engines_load_balance) + bond_count * sizeof(bond_ci);
4970 balancer = (struct i915_context_engines_load_balance*)malloc(size);
4971 if (NULL == balancer)
4972 {
4973 ret = -ENOMEM;
4974 goto fini;
4975 }
4976 memset(balancer, 0, size);
4977 balancer->base.name = I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE;
4978 balancer->num_siblings = bond_count;
4979 memcpy(balancer->engines, bond_ci, bond_count * sizeof(*bond_ci));
4980
4981 /* I915_DEFINE_CONTEXT_ENGINES_BOND */
4982 size = sizeof(struct i915_context_engines_bond) + bond_count * sizeof(*bond_ci);
4983 bond = (struct i915_context_engines_bond*)malloc(size);
4984 if (NULL == bond)
4985 {
4986 ret = -ENOMEM;
4987 goto fini;
4988 }
4989 memset(bond, 0, size);
4990 bond->base.name = I915_CONTEXT_ENGINES_EXT_BOND;
4991 bond->master = master_ci;
4992 bond->num_bonds = bond_count;
4993 memcpy(bond->engines, bond_ci, bond_count * sizeof(*bond_ci));
4994
4995 /* I915_DEFINE_CONTEXT_PARAM_ENGINES */
4996 size = sizeof(uint64_t) + sizeof(struct i915_engine_class_instance);
4997 set_engines = (struct i915_context_param_engines*) malloc(size);
4998 if (NULL == set_engines)
4999 {
5000 ret = -ENOMEM;
5001 goto fini;
5002 }
5003 set_engines->extensions = (uintptr_t)(balancer);
5004 balancer->base.next_extension = (uintptr_t)(bond);
5005 set_engines->engines[0].engine_class = I915_ENGINE_CLASS_INVALID;
5006 set_engines->engines[0].engine_instance = I915_ENGINE_CLASS_INVALID_NONE;
5007
5008 ret = mos_set_context_param(ctx,
5009 size,
5010 I915_CONTEXT_PARAM_ENGINES,
5011 (uintptr_t)set_engines);
5012 fini:
5013 if (set_engines)
5014 free(set_engines);
5015 if (bond)
5016 free(bond);
5017 if (balancer)
5018 free(balancer);
5019 return ret;
5020 }
5021
mos_gem_bo_is_softpin(struct mos_linux_bo * bo)5022 static bool mos_gem_bo_is_softpin(struct mos_linux_bo *bo)
5023 {
5024 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *) bo;
5025 if (bo_gem == nullptr)
5026 {
5027 return false;
5028 }
5029
5030 return bo_gem->is_softpin;
5031 }
5032
5033 static bool
mos_gem_bo_is_exec_object_async(struct mos_linux_bo * bo)5034 mos_gem_bo_is_exec_object_async(struct mos_linux_bo *bo)
5035 {
5036 struct mos_bo_gem *bo_gem = (struct mos_bo_gem *)bo;
5037
5038 return bo_gem->exec_async;
5039 }
5040
5041
mos_bufmgr_query_device_blob(struct mos_bufmgr * bufmgr,MEDIA_SYSTEM_INFO * gfx_info)5042 static int mos_bufmgr_query_device_blob(struct mos_bufmgr *bufmgr, MEDIA_SYSTEM_INFO* gfx_info)
5043 {
5044 if ((bufmgr == nullptr) || (gfx_info == nullptr))
5045 {
5046 return -EINVAL;
5047 }
5048
5049 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5050 if (bufmgr_gem == nullptr)
5051 {
5052 return -EINVAL;
5053 }
5054 int fd = bufmgr_gem->fd;
5055 uint32_t *hw_info = nullptr;
5056 uint32_t ulength= 0;
5057 int ret, i;
5058 struct drm_i915_query_item query_item;
5059
5060 memclear(query_item);
5061 query_item.length = 0;
5062 query_item.query_id = DRM_I915_QUERY_HWCONFIG_BLOB;
5063 ret = mos_gem_query_items(fd, &query_item, 1);
5064 if (ret != 0 || query_item.length <= 0)
5065 {
5066 mos_safe_free(hw_info);
5067 return (ret != 0) ? ret : -1;
5068 }
5069
5070 hw_info = (uint32_t*) malloc(query_item.length);
5071 if (hw_info != nullptr)
5072 {
5073 memset(hw_info, 0, query_item.length);
5074 }
5075 else
5076 {
5077 mos_safe_free(hw_info);
5078 return -ENOMEM;
5079 }
5080 query_item.data_ptr = (uintptr_t) hw_info;
5081 ret = mos_gem_query_items(fd, &query_item, 1);
5082 if (ret != 0 || query_item.length <= 0)
5083 {
5084 mos_safe_free(hw_info);
5085 return (ret != 0) ? ret : -1;
5086 }
5087 ulength = query_item.length / sizeof(uint32_t);
5088 i = 0;
5089 while (i < ulength) {
5090 /* Attribute ID starts with 1 */
5091 if (hw_info[i] <= 0)
5092 {
5093 mos_safe_free(hw_info);
5094 return -EINVAL;
5095 }
5096
5097 #if DEBUG_BLOB_QUERY
5098 fprintf("%s: %d\n", key_string[hw_info[i]], hw_info[i+2]);
5099 #endif
5100 if (INTEL_HWCONFIG_MAX_SLICES_SUPPORTED == hw_info[i])
5101 {
5102 if (hw_info[i+1] != 1)
5103 {
5104 mos_safe_free(hw_info);
5105 return -EINVAL;
5106 }
5107 gfx_info->SliceCount = hw_info[i+2];
5108 gfx_info->MaxSlicesSupported = hw_info[i+2];
5109 }
5110
5111 if (INTEL_HWCONFIG_MAX_DUAL_SUBSLICES_SUPPORTED == hw_info[i])
5112 {
5113 if (hw_info[i+1] != 1)
5114 {
5115 mos_safe_free(hw_info);
5116 return -EINVAL;
5117 }
5118 gfx_info->SubSliceCount = hw_info[i+2];
5119 gfx_info->MaxSubSlicesSupported = hw_info[i+2];
5120 }
5121
5122 if (INTEL_HWCONFIG_MAX_NUM_EU_PER_DSS == hw_info[i])
5123 {
5124 if (hw_info[i+1] != 1)
5125 {
5126 mos_safe_free(hw_info);
5127 return -EINVAL;
5128 }
5129 gfx_info->MaxEuPerSubSlice = hw_info[i+2];
5130 }
5131
5132 if (INTEL_HWCONFIG_DEPRECATED_L3_CACHE_SIZE_IN_KB == hw_info[i])
5133 {
5134 if (hw_info[i+1] != 1)
5135 {
5136 mos_safe_free(hw_info);
5137 return -EINVAL;
5138 }
5139 gfx_info->L3CacheSizeInKb = hw_info[i+2];
5140 }
5141
5142 if (INTEL_HWCONFIG_NUM_THREADS_PER_EU == hw_info[i])
5143 {
5144 if (hw_info[i+1] != 1)
5145 {
5146 mos_safe_free(hw_info);
5147 return -EINVAL;
5148 }
5149 gfx_info->NumThreadsPerEu = hw_info[i+2];
5150 }
5151
5152 if (INTEL_HWCONFIG_MAX_VECS == hw_info[i])
5153 {
5154 if (hw_info[i+1] != 1)
5155 {
5156 mos_safe_free(hw_info);
5157 return -EINVAL;
5158 }
5159 gfx_info->MaxVECS = hw_info[i+2];
5160 }
5161
5162 /* Advance to next key */
5163 i += hw_info[i + 1]; // value size
5164 i += 2;// KL size
5165 }
5166
5167 if (hw_info != nullptr)
5168 {
5169 mos_safe_free(hw_info);
5170 hw_info = nullptr;
5171 }
5172
5173 return ret;
5174 }
5175
mos_bufmgr_query_hw_ip_version(struct mos_bufmgr * bufmgr,__u16 engine_class,void * ip_ver_info)5176 static int mos_bufmgr_query_hw_ip_version(struct mos_bufmgr *bufmgr, __u16 engine_class, void *ip_ver_info)
5177 {
5178 return -1;
5179 }
5180
mos_bufmgr_get_platform_information(struct mos_bufmgr * bufmgr)5181 static uint64_t mos_bufmgr_get_platform_information(struct mos_bufmgr *bufmgr)
5182 {
5183 assert(bufmgr);
5184 return bufmgr->platform_information;
5185 }
5186
mos_bufmgr_set_platform_information(struct mos_bufmgr * bufmgr,uint64_t p)5187 static void mos_bufmgr_set_platform_information(struct mos_bufmgr *bufmgr, uint64_t p)
5188 {
5189 assert(bufmgr);
5190 bufmgr->platform_information |= p;
5191 }
5192
5193 static int
mos_bufmgr_get_ts_frequency(struct mos_bufmgr * bufmgr,uint32_t * ts_freq)5194 mos_bufmgr_get_ts_frequency(struct mos_bufmgr *bufmgr, uint32_t *ts_freq)
5195 {
5196 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5197
5198 *ts_freq = bufmgr_gem->ts_freq;
5199
5200 return 0;
5201 }
5202
5203 static bool
mos_bufmgr_has_bsd2(struct mos_bufmgr * bufmgr)5204 mos_bufmgr_has_bsd2(struct mos_bufmgr *bufmgr)
5205 {
5206 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5207
5208 return bufmgr_gem->has_bsd2;
5209 }
5210
5211 #define I915_CONTEXT_PRIVATE_PARAM_BOOST 0x80000000
5212 void
mos_bufmgr_enable_turbo_boost(struct mos_bufmgr * bufmgr)5213 mos_bufmgr_enable_turbo_boost(struct mos_bufmgr *bufmgr)
5214 {
5215 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5216 struct drm_i915_gem_context_param ctxParam;
5217 int32_t retVal = 0;
5218
5219 MOS_ZeroMemory( &ctxParam, sizeof( ctxParam ) );
5220 ctxParam.param = I915_CONTEXT_PRIVATE_PARAM_BOOST;
5221 ctxParam.value = 1;
5222 retVal = drmIoctl(bufmgr_gem->fd,
5223 DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &ctxParam );
5224 }
5225
5226 /**
5227 * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
5228 * and manage map buffer objections.
5229 *
5230 * \param fd File descriptor of the opened DRM device.
5231 */
5232 static struct mos_bufmgr *
mos_bufmgr_gem_init_i915(int fd,int batch_size)5233 mos_bufmgr_gem_init_i915(int fd, int batch_size)
5234 {
5235 struct mos_bufmgr_gem *bufmgr_gem;
5236 struct drm_i915_gem_get_aperture aperture;
5237 drm_i915_getparam_t gp;
5238 int ret, tmp;
5239 uint8_t alloc_mode;
5240 bool exec2 = false;
5241
5242 pthread_mutex_lock(&bufmgr_list_mutex);
5243
5244 bufmgr_gem = mos_bufmgr_gem_find(fd);
5245 if (bufmgr_gem)
5246 goto exit;
5247
5248 bufmgr_gem = (struct mos_bufmgr_gem *)calloc(1, sizeof(*bufmgr_gem));
5249 if (bufmgr_gem == nullptr)
5250 goto exit;
5251
5252 bufmgr_gem->fd = fd;
5253 atomic_set(&bufmgr_gem->refcount, 1);
5254
5255 if (pthread_mutex_init(&bufmgr_gem->lock, nullptr) != 0) {
5256 free(bufmgr_gem);
5257 bufmgr_gem = nullptr;
5258 goto exit;
5259 }
5260
5261 bufmgr_gem->bufmgr.bo_alloc = mos_gem_bo_alloc;
5262 bufmgr_gem->bufmgr.bo_alloc_tiled = mos_gem_bo_alloc_tiled;
5263 bufmgr_gem->bufmgr.bo_reference = mos_gem_bo_reference;
5264 bufmgr_gem->bufmgr.bo_unreference = mos_gem_bo_unreference;
5265 bufmgr_gem->bufmgr.bo_map = mos_gem_bo_map;
5266 bufmgr_gem->bufmgr.bo_unmap = mos_gem_bo_unmap;
5267 bufmgr_gem->bufmgr.bo_wait_rendering = mos_gem_bo_wait_rendering;
5268 bufmgr_gem->bufmgr.bo_pad_to_size = mos_gem_bo_pad_to_size;
5269 bufmgr_gem->bufmgr.bo_emit_reloc = mos_gem_bo_emit_reloc;
5270 bufmgr_gem->bufmgr.bo_get_tiling = mos_gem_bo_get_tiling;
5271 bufmgr_gem->bufmgr.bo_set_tiling = mos_gem_bo_set_tiling;
5272 bufmgr_gem->bufmgr.bo_flink = mos_gem_bo_flink;
5273 bufmgr_gem->bufmgr.bo_exec = mos_gem_bo_exec2;
5274 bufmgr_gem->bufmgr.bo_mrb_exec = mos_gem_bo_mrb_exec2;
5275 bufmgr_gem->bufmgr.bo_busy = mos_gem_bo_busy;
5276 bufmgr_gem->bufmgr.bo_madvise = mos_gem_bo_madvise;
5277 bufmgr_gem->bufmgr.destroy = mos_bufmgr_gem_unref;
5278 bufmgr_gem->bufmgr.debug = 0;
5279 bufmgr_gem->bufmgr.check_aperture_space =
5280 mos_gem_check_aperture_space;
5281 bufmgr_gem->bufmgr.bo_disable_reuse = mos_gem_bo_disable_reuse;
5282 bufmgr_gem->bufmgr.bo_is_reusable = mos_gem_bo_is_reusable;
5283 bufmgr_gem->bufmgr.bo_references = mos_gem_bo_references;
5284
5285 bufmgr_gem->bufmgr.bo_wait = mos_gem_bo_wait;
5286 bufmgr_gem->bufmgr.bo_clear_relocs = mos_gem_bo_clear_relocs;
5287 bufmgr_gem->bufmgr.context_create = mos_gem_context_create;
5288 bufmgr_gem->bufmgr.context_create_ext = mos_gem_context_create_ext;
5289 bufmgr_gem->bufmgr.context_create_shared = mos_gem_context_create_shared;
5290 bufmgr_gem->bufmgr.context_destroy = mos_gem_context_destroy;
5291 bufmgr_gem->bufmgr.vm_create = mos_gem_vm_create;
5292 bufmgr_gem->bufmgr.vm_destroy = mos_gem_vm_destroy;
5293 bufmgr_gem->bufmgr.bo_context_exec2 = mos_gem_bo_context_exec2;
5294 bufmgr_gem->bufmgr.bo_context_exec3 = mos_gem_bo_context_exec3;
5295 bufmgr_gem->bufmgr.bo_is_exec_object_async = mos_gem_bo_is_exec_object_async;
5296 bufmgr_gem->bufmgr.bo_is_softpin = mos_gem_bo_is_softpin;
5297 bufmgr_gem->bufmgr.bo_map_gtt = mos_gem_bo_map_gtt;
5298 bufmgr_gem->bufmgr.bo_unmap_gtt = mos_gem_bo_unmap_gtt;
5299 bufmgr_gem->bufmgr.bo_map_wc = mos_gem_bo_map_wc;
5300 bufmgr_gem->bufmgr.bo_unmap_wc = mos_gem_bo_unmap_wc;
5301 bufmgr_gem->bufmgr.bo_map_unsynchronized = mos_gem_bo_map_unsynchronized;
5302 bufmgr_gem->bufmgr.bo_start_gtt_access = mos_gem_bo_start_gtt_access;
5303 bufmgr_gem->bufmgr.bo_get_softpin_targets_info = mos_bufmgr_bo_get_softpin_targets_info;
5304 bufmgr_gem->bufmgr.bo_create_from_name = mos_bufmgr_bo_gem_create_from_name;
5305 bufmgr_gem->bufmgr.enable_reuse = mos_gem_enable_reuse;
5306 bufmgr_gem->bufmgr.enable_softpin = mos_gem_enable_softpin;
5307 bufmgr_gem->bufmgr.enable_vmbind = mos_gem_enable_vmbind;
5308 bufmgr_gem->bufmgr.disable_object_capture = mos_gem_disable_object_capture;
5309 bufmgr_gem->bufmgr.get_memory_info = mos_gem_get_memory_info;
5310 bufmgr_gem->bufmgr.get_devid = mos_gem_get_devid;
5311 bufmgr_gem->bufmgr.realloc_cache = mos_gem_realloc_cache;
5312 bufmgr_gem->bufmgr.set_context_param = mos_gem_set_context_param;
5313 bufmgr_gem->bufmgr.set_context_param_parallel = mos_gem_set_context_param_parallel;
5314 bufmgr_gem->bufmgr.set_context_param_load_balance = mos_gem_set_context_param_load_balance;
5315 bufmgr_gem->bufmgr.set_context_param_bond = mos_gem_set_context_param_bond;
5316 bufmgr_gem->bufmgr.get_context_param = mos_gem_get_context_param;
5317 bufmgr_gem->bufmgr.bo_create_from_prime = mos_gem_bo_create_from_prime;
5318 bufmgr_gem->bufmgr.bo_export_to_prime = mos_gem_bo_export_to_prime;
5319 bufmgr_gem->bufmgr.reg_read = mos_bufmg_reg_read;
5320 bufmgr_gem->bufmgr.get_reset_stats = mos_bufmg_get_reset_stats;
5321 bufmgr_gem->bufmgr.get_context_param_sseu = mos_bufmgr_get_context_param_sseu;
5322 bufmgr_gem->bufmgr.set_context_param_sseu = mos_bufmgr_set_context_param_sseu;
5323 bufmgr_gem->bufmgr.query_sys_engines = mos_bufmgr_query_sys_engines;
5324 bufmgr_gem->bufmgr.query_device_blob = mos_bufmgr_query_device_blob;
5325 bufmgr_gem->bufmgr.get_driver_info = mos_bufmgr_get_driver_info;
5326 bufmgr_gem->bufmgr.query_hw_ip_version = mos_bufmgr_query_hw_ip_version;
5327 bufmgr_gem->bufmgr.get_platform_information = mos_bufmgr_get_platform_information;
5328 bufmgr_gem->bufmgr.set_platform_information = mos_bufmgr_set_platform_information;
5329 bufmgr_gem->bufmgr.query_engines_count = mos_bufmgr_query_engines_count;
5330 bufmgr_gem->bufmgr.query_engines = mos_bufmgr_query_engines;
5331 bufmgr_gem->bufmgr.get_engine_class_size = mos_bufmgr_get_engine_class_size;
5332 bufmgr_gem->bufmgr.select_fixed_engine = mos_gem_select_fixed_engine;
5333 bufmgr_gem->bufmgr.switch_off_n_bits = mos_bufmgr_switch_off_n_bits;
5334 bufmgr_gem->bufmgr.hweight8 = mos_bufmgr_hweight8;
5335 bufmgr_gem->bufmgr.get_ts_frequency = mos_bufmgr_get_ts_frequency;
5336 bufmgr_gem->bufmgr.has_bsd2 = mos_bufmgr_has_bsd2;
5337 bufmgr_gem->bufmgr.enable_turbo_boost = mos_bufmgr_enable_turbo_boost;
5338
5339 bufmgr_gem->mem_profiler_path = getenv("MEDIA_MEMORY_PROFILER_LOG");
5340 if (bufmgr_gem->mem_profiler_path != nullptr)
5341 {
5342 if (strcmp(bufmgr_gem->mem_profiler_path, "/sys/kernel/debug/tracing/trace_marker") == 0)
5343 {
5344 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY );
5345 }
5346 else
5347 {
5348 ret = bufmgr_gem->mem_profiler_fd = open(bufmgr_gem->mem_profiler_path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR);
5349 }
5350
5351 if (ret == -1)
5352 {
5353 MOS_DBG("Failed to open %s: %s\n", bufmgr_gem->mem_profiler_path, strerror(errno));
5354 }
5355 }
5356 else
5357 {
5358 bufmgr_gem->mem_profiler_fd = -1;
5359 }
5360
5361 memclear(aperture);
5362 ret = drmIoctl(bufmgr_gem->fd,
5363 DRM_IOCTL_I915_GEM_GET_APERTURE,
5364 &aperture);
5365
5366 if (ret == 0)
5367 bufmgr_gem->gtt_size = aperture.aper_available_size;
5368 else {
5369 fprintf(stderr, "DRM_IOCTL_I915_GEM_APERTURE failed: %s\n",
5370 strerror(errno));
5371 bufmgr_gem->gtt_size = 128 * 1024 * 1024;
5372 fprintf(stderr, "Assuming %dkB available aperture size.\n"
5373 "May lead to reduced performance or incorrect "
5374 "rendering.\n",
5375 (int)bufmgr_gem->gtt_size / 1024);
5376 }
5377
5378 /* support Gen 8+ */
5379 bufmgr_gem->pci_device = get_pci_device_id(bufmgr_gem);
5380
5381 if (bufmgr_gem->pci_device == 0) {
5382 pthread_mutex_destroy(&bufmgr_gem->lock);
5383 if (bufmgr_gem->mem_profiler_fd != -1)
5384 {
5385 close(bufmgr_gem->mem_profiler_fd);
5386 }
5387 free(bufmgr_gem);
5388 bufmgr_gem = nullptr;
5389 goto exit;
5390 }
5391
5392 memclear(gp);
5393 gp.value = &tmp;
5394
5395 gp.param = I915_PARAM_HAS_EXECBUF2;
5396 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5397 if (!ret)
5398 exec2 = true;
5399
5400 gp.param = I915_PARAM_HAS_BSD;
5401 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5402 bufmgr_gem->has_bsd = ret == 0;
5403
5404 gp.param = I915_PARAM_HAS_BSD2;
5405 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5406 bufmgr_gem->has_bsd2 = (ret == 0 && *gp.value != 0);
5407
5408 gp.param = I915_PARAM_HAS_BLT;
5409 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5410 bufmgr_gem->has_blt = ret == 0;
5411
5412 gp.param = I915_PARAM_HAS_RELAXED_FENCING;
5413 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5414 bufmgr_gem->has_relaxed_fencing = ret == 0;
5415
5416 bufmgr_gem->bufmgr.bo_alloc_userptr = check_bo_alloc_userptr;
5417
5418 gp.param = I915_PARAM_HAS_WAIT_TIMEOUT;
5419 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5420 bufmgr_gem->has_wait_timeout = ret == 0;
5421
5422 gp.param = I915_PARAM_HAS_LLC;
5423 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5424 bufmgr_gem->has_llc = *gp.value;
5425
5426 gp.param = I915_PARAM_HAS_VEBOX;
5427 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5428 bufmgr_gem->has_vebox = (ret == 0) & (*gp.value > 0);
5429
5430 gp.param = I915_PARAM_MMAP_VERSION;
5431 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5432 bufmgr_gem->has_ext_mmap = (ret == 0) & (*gp.value > 0);
5433
5434 gp.param = I915_PARAM_NUM_FENCES_AVAIL;
5435 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5436 bufmgr_gem->has_fence_reg = (ret == 0) & (*gp.value > 0);
5437
5438 gp.param = I915_PARAM_HAS_EXEC_SOFTPIN;
5439 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5440 if (ret == 0 && *gp.value > 0) {
5441 bufmgr_gem->bufmgr.bo_set_softpin = mos_gem_bo_set_softpin;
5442 bufmgr_gem->bufmgr.bo_add_softpin_target = mos_gem_bo_add_softpin_target;
5443 }
5444
5445 gp.param = I915_PARAM_HAS_EXEC_ASYNC;
5446 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5447 if (ret == 0 && *gp.value > 0) {
5448 bufmgr_gem->bufmgr.set_object_async = mos_gem_bo_set_object_async;
5449 bufmgr_gem->bufmgr.set_exec_object_async = mos_gem_bo_set_exec_object_async;
5450 }
5451
5452 gp.param = I915_PARAM_HAS_EXEC_CAPTURE;
5453 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GETPARAM, &gp);
5454 if (ret == 0 && *gp.value > 0)
5455 {
5456 bufmgr_gem->bufmgr.set_object_capture = mos_gem_bo_set_object_capture;
5457 }
5458 else
5459 {
5460 bufmgr_gem->object_capture_disabled = true;
5461 }
5462
5463 gp.param = I915_PARAM_MMAP_GTT_VERSION;
5464 ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
5465 bufmgr_gem->has_mmap_offset = (ret == 0) && (*gp.value >= 4);
5466
5467 gp.param = I915_PARAM_CS_TIMESTAMP_FREQUENCY;
5468 ret = drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp);
5469 bufmgr_gem->ts_freq = (ret == 0) ? *gp.value : 0;
5470
5471 struct drm_i915_gem_context_param context_param;
5472 memset(&context_param, 0, sizeof(context_param));
5473 context_param.param = I915_CONTEXT_PARAM_GTT_SIZE;
5474 ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &context_param);
5475 if (ret == 0){
5476 uint64_t gtt_size_4g = (uint64_t)1 << 32;
5477 if (context_param.value > gtt_size_4g)
5478 {
5479 bufmgr_gem->bufmgr.bo_use_48b_address_range = mos_gem_bo_use_48b_address_range;
5480 }
5481 }
5482
5483 bufmgr_gem->device_type = DEVICE_TYPE_I915;
5484 bufmgr_gem->has_lmem = mos_gem_has_lmem(bufmgr_gem->fd);
5485
5486 bufmgr_gem->bufmgr.has_full_vd = true;
5487
5488 /* Let's go with one relocation per every 2 dwords (but round down a bit
5489 * since a power of two will mean an extra page allocation for the reloc
5490 * buffer).
5491 *
5492 * Every 4 was too few for the blender benchmark.
5493 */
5494 alloc_mode = (uint8_t)(batch_size & 0xff);
5495 batch_size &= 0xffffff00;
5496 bufmgr_gem->max_relocs = batch_size / sizeof(uint32_t) / 2 - 2;
5497
5498 DRMINITLISTHEAD(&bufmgr_gem->named);
5499 init_cache_buckets(bufmgr_gem);
5500
5501 DRMLISTADD(&bufmgr_gem->managers, &bufmgr_list);
5502
5503 bufmgr_gem->use_softpin = false;
5504 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_SYS], MEMZONE_SYS_START, MEMZONE_SYS_SIZE);
5505 mos_vma_heap_init(&bufmgr_gem->vma_heap[MEMZONE_DEVICE], MEMZONE_DEVICE_START, MEMZONE_DEVICE_SIZE);
5506
5507 exit:
5508 pthread_mutex_unlock(&bufmgr_list_mutex);
5509
5510 return bufmgr_gem != nullptr ? &bufmgr_gem->bufmgr : nullptr;
5511 }
5512
5513 struct mos_bufmgr *
mos_bufmgr_gem_init(int fd,int batch_size,int * device_type)5514 mos_bufmgr_gem_init(int fd, int batch_size, int *device_type)
5515 {
5516 int type = mos_query_device_type(fd);
5517 if (device_type != nullptr)
5518 {
5519 *device_type = type;
5520 }
5521 if(DEVICE_TYPE_I915 == type)
5522 {
5523 return mos_bufmgr_gem_init_i915(fd, batch_size);
5524 }
5525 #ifdef ENABLE_XE_KMD
5526 else if (DEVICE_TYPE_XE == type)
5527 {
5528 return mos_bufmgr_gem_init_xe(fd, batch_size);
5529 }
5530 #endif
5531
5532 return nullptr;
5533 }
5534
mos_get_param(int fd,int32_t param,uint32_t * param_value)5535 int mos_get_param(int fd, int32_t param, uint32_t *param_value)
5536 {
5537 if((fd < 0) || (param_value == nullptr))
5538 {
5539 return -EINVAL;
5540 }
5541
5542 struct drm_i915_getparam gp;
5543 gp.param = param;
5544 gp.value = (int32_t *)param_value;
5545
5546 return drmIoctl(fd, DRM_IOCTL_I915_GETPARAM, &gp) == 0;
5547 }
5548
mos_bufmgr_get_driver_info(struct mos_bufmgr * bufmgr,struct LinuxDriverInfo * drvInfo)5549 static int mos_bufmgr_get_driver_info(struct mos_bufmgr *bufmgr, struct LinuxDriverInfo *drvInfo)
5550 {
5551 if (bufmgr == nullptr || drvInfo == nullptr)
5552 {
5553 return -EINVAL;
5554 }
5555 struct mos_bufmgr_gem *bufmgr_gem = (struct mos_bufmgr_gem*)bufmgr;
5556 int fd = bufmgr_gem->fd;
5557 if (fd < 0)
5558 {
5559 return -EINVAL;
5560 }
5561 uint32_t retValue = 0;
5562
5563 drvInfo->hasBsd = 0;
5564 if (mos_get_param(fd, I915_PARAM_HAS_BSD, &retValue))
5565 {
5566 drvInfo->hasBsd = !!retValue;
5567 }
5568
5569 drvInfo->hasBsd2 = 0;
5570 retValue = 0;
5571 if (mos_get_param(fd, I915_PARAM_HAS_BSD2, &retValue))
5572 {
5573 drvInfo->hasBsd2 = !!retValue;
5574 }
5575
5576 drvInfo->hasVebox = 0;
5577 retValue = 0;
5578 if (mos_get_param(fd, I915_PARAM_HAS_VEBOX, &retValue))
5579 {
5580 drvInfo->hasVebox = !!retValue;
5581 }
5582
5583 drvInfo->hasPpgtt = 1;
5584 retValue = 0;
5585 if (mos_get_param(fd, I915_PARAM_HAS_ALIASING_PPGTT, &retValue))
5586 {
5587 drvInfo->hasPpgtt = !!retValue;
5588 }
5589
5590 drvInfo->hasHuc = 0;
5591 retValue = 0;
5592 if (mos_get_param(fd, I915_PARAM_HUC_STATUS, &retValue))
5593 {
5594 drvInfo->hasHuc = !!retValue;
5595 if (retValue == 1)
5596 {
5597 drvInfo->hasProtectedHuc = 1;
5598 }
5599 }
5600
5601 drvInfo->devId = 0;
5602 retValue = 0;
5603 if (mos_get_param(fd, I915_PARAM_CHIPSET_ID, &retValue))
5604 {
5605 drvInfo->devId = retValue;
5606 }
5607 drvInfo->devRev = 0;
5608 retValue = 0;
5609 if (mos_get_param(fd, I915_PARAM_REVISION, &retValue))
5610 {
5611 drvInfo->devRev = retValue;
5612 }
5613
5614 drvInfo->euCount = 0;
5615 retValue = 0;
5616 if (mos_get_param(fd, I915_PARAM_EU_TOTAL, &retValue))
5617 {
5618 drvInfo->euCount = retValue;
5619 }
5620
5621 drvInfo->subSliceCount = 0;
5622 retValue = 0;
5623 if (mos_get_param(fd, I915_PARAM_SUBSLICE_TOTAL, &retValue))
5624 {
5625 drvInfo->subSliceCount = retValue;
5626 }
5627
5628 // There is no interface to read total slice count from drm/i915, so we
5629 // will set the slice count in InitMediaSysInfo accordint to Device ID.
5630 drvInfo->sliceCount = 0;
5631
5632 return 0;
5633 }
5634
mos_get_dev_id_i915(int fd,uint32_t * device_id)5635 static int mos_get_dev_id_i915(int fd, uint32_t *device_id)
5636 {
5637 if (nullptr == device_id)
5638 {
5639 return -EINVAL;
5640 }
5641 uint32_t retValue = 0;
5642 if (mos_get_param(fd, I915_PARAM_CHIPSET_ID, &retValue))
5643 {
5644 *device_id = retValue;
5645 }
5646
5647 return 0;
5648 }
5649
mos_get_device_id(int fd,uint32_t * deviceId)5650 int mos_get_device_id(int fd, uint32_t *deviceId)
5651 {
5652 int device_type = mos_query_device_type(fd);
5653
5654 if (DEVICE_TYPE_I915 == device_type)
5655 {
5656 return mos_get_dev_id_i915(fd, deviceId);
5657 }
5658 #ifdef ENABLE_XE_KMD
5659 else if (DEVICE_TYPE_XE == device_type)
5660 {
5661 return mos_get_dev_id_xe(fd, deviceId);
5662 }
5663 #endif
5664 return -ENODEV;
5665 }
5666