1 /*
2 * Copyright © 2022 Google, Inc.
3 * SPDX-License-Identifier: MIT
4 */
5
6 #include <assert.h>
7 #include <inttypes.h>
8 #include <pthread.h>
9
10 #include "util/libsync.h"
11 #include "util/os_file.h"
12
13 #include "drm/freedreno_ringbuffer_sp.h"
14 #include "virtio_priv.h"
15
16 static void
retire_execute(void * job,void * gdata,int thread_index)17 retire_execute(void *job, void *gdata, int thread_index)
18 {
19 struct fd_submit_sp *fd_submit = job;
20
21 MESA_TRACE_FUNC();
22
23 fd_fence_wait(fd_submit->out_fence);
24 }
25
26 static void
retire_cleanup(void * job,void * gdata,int thread_index)27 retire_cleanup(void *job, void *gdata, int thread_index)
28 {
29 struct fd_submit_sp *fd_submit = job;
30 fd_submit_del(&fd_submit->base);
31 }
32
33 static int
flush_submit_list(struct list_head * submit_list)34 flush_submit_list(struct list_head *submit_list)
35 {
36 struct fd_submit_sp *fd_submit = to_fd_submit_sp(last_submit(submit_list));
37 struct virtio_pipe *virtio_pipe = to_virtio_pipe(fd_submit->base.pipe);
38 struct fd_pipe *pipe = &virtio_pipe->base;
39 struct fd_device *dev = pipe->dev;
40
41 unsigned nr_cmds = 0;
42
43 MESA_TRACE_FUNC();
44
45 /* Determine the number of extra cmds's from deferred submits that
46 * we will be merging in:
47 */
48 foreach_submit (submit, submit_list) {
49 assert(submit->pipe == &virtio_pipe->base);
50 nr_cmds += to_fd_ringbuffer_sp(submit->primary)->u.nr_cmds;
51 }
52
53 /* TODO we can get rid of the extra copy into the req by just
54 * assuming the max amount that nr->bos will grow is by the
55 * nr_cmds, and just over-allocate a bit.
56 */
57
58 struct drm_msm_gem_submit_cmd cmds[nr_cmds];
59
60 unsigned cmd_idx = 0;
61
62 /* Build up the table of cmds, and for all but the last submit in the
63 * list, merge their bo tables into the last submit.
64 */
65 foreach_submit_safe (submit, submit_list) {
66 struct fd_ringbuffer_sp *deferred_primary =
67 to_fd_ringbuffer_sp(submit->primary);
68
69 for (unsigned i = 0; i < deferred_primary->u.nr_cmds; i++) {
70 struct fd_bo *ring_bo = deferred_primary->u.cmds[i].ring_bo;
71 cmds[cmd_idx].type = MSM_SUBMIT_CMD_BUF;
72 cmds[cmd_idx].submit_idx = fd_submit_append_bo(fd_submit, ring_bo);
73 cmds[cmd_idx].submit_offset = submit_offset(ring_bo, deferred_primary->offset);
74 cmds[cmd_idx].size = deferred_primary->u.cmds[i].size;
75 cmds[cmd_idx].pad = 0;
76 cmds[cmd_idx].nr_relocs = 0;
77
78 cmd_idx++;
79 }
80
81 /* We are merging all the submits in the list into the last submit,
82 * so the remainder of the loop body doesn't apply to the last submit
83 */
84 if (submit == last_submit(submit_list)) {
85 DEBUG_MSG("merged %u submits", cmd_idx);
86 break;
87 }
88
89 struct fd_submit_sp *fd_deferred_submit = to_fd_submit_sp(submit);
90 for (unsigned i = 0; i < fd_deferred_submit->nr_bos; i++) {
91 /* Note: if bo is used in both the current submit and the deferred
92 * submit being merged, we expect to hit the fast-path as we add it
93 * to the current submit:
94 */
95 fd_submit_append_bo(fd_submit, fd_deferred_submit->bos[i]);
96 }
97
98 /* Now that the cmds/bos have been transfered over to the current submit,
99 * we can remove the deferred submit from the list and drop it's reference
100 */
101 list_del(&submit->node);
102 fd_submit_del(submit);
103 }
104
105 /* Needs to be after get_cmd() as that could create bos/cmds table:
106 *
107 * NOTE allocate on-stack in the common case, but with an upper-
108 * bound to limit on-stack allocation to 4k:
109 */
110 const unsigned bo_limit = 4096 / sizeof(struct drm_msm_gem_submit_bo);
111 bool bos_on_stack = fd_submit->nr_bos < bo_limit;
112 struct drm_msm_gem_submit_bo
113 _submit_bos[bos_on_stack ? fd_submit->nr_bos : 0];
114 struct drm_msm_gem_submit_bo *submit_bos;
115 uint32_t _guest_handles[bos_on_stack ? fd_submit->nr_bos : 0];
116 uint32_t *guest_handles;
117 if (bos_on_stack) {
118 submit_bos = _submit_bos;
119 guest_handles = _guest_handles;
120 } else {
121 submit_bos = malloc(fd_submit->nr_bos * sizeof(submit_bos[0]));
122 guest_handles = malloc(fd_submit->nr_bos * sizeof(guest_handles[0]));
123 }
124
125 uint32_t nr_guest_handles = 0;
126 for (unsigned i = 0; i < fd_submit->nr_bos; i++) {
127 struct virtio_bo *virtio_bo = to_virtio_bo(fd_submit->bos[i]);
128
129 if (virtio_bo->base.alloc_flags & FD_BO_SHARED)
130 guest_handles[nr_guest_handles++] = virtio_bo->base.handle;
131
132 submit_bos[i].flags = fd_submit->bos[i]->reloc_flags;
133 submit_bos[i].handle = virtio_bo->res_id;
134 submit_bos[i].presumed = 0;
135 }
136
137 if (virtio_pipe->next_submit_fence <= 0)
138 virtio_pipe->next_submit_fence = 1;
139
140 uint32_t kfence = virtio_pipe->next_submit_fence++;
141
142 /* TODO avoid extra memcpy, and populate bo's and cmds directly
143 * into the req msg
144 */
145 unsigned bos_len = fd_submit->nr_bos * sizeof(struct drm_msm_gem_submit_bo);
146 unsigned cmd_len = nr_cmds * sizeof(struct drm_msm_gem_submit_cmd);
147 unsigned req_len = sizeof(struct msm_ccmd_gem_submit_req) + bos_len + cmd_len;
148 struct msm_ccmd_gem_submit_req *req = malloc(req_len);
149
150 req->hdr = MSM_CCMD(GEM_SUBMIT, req_len);
151 req->flags = virtio_pipe->pipe;
152 req->queue_id = virtio_pipe->queue_id;
153 req->nr_bos = fd_submit->nr_bos;
154 req->nr_cmds = nr_cmds;
155 req->fence = kfence;
156
157 memcpy(req->payload, submit_bos, bos_len);
158 memcpy(req->payload + bos_len, cmds, cmd_len);
159
160 struct fd_fence *out_fence = fd_submit->out_fence;
161
162 out_fence->kfence = kfence;
163
164 /* Even if gallium driver hasn't requested a fence-fd, request one.
165 * This way, if we have to block waiting for the fence, we can do
166 * it in the guest, rather than in the single-threaded host.
167 */
168 out_fence->use_fence_fd = true;
169
170 if (pipe->no_implicit_sync) {
171 req->flags |= MSM_SUBMIT_NO_IMPLICIT;
172 nr_guest_handles = 0;
173 }
174
175 struct vdrm_execbuf_params p = {
176 .req = &req->hdr,
177 .handles = guest_handles,
178 .num_handles = nr_guest_handles,
179 .has_in_fence_fd = !!(fd_submit->in_fence_fd != -1),
180 .needs_out_fence_fd = true,
181 .fence_fd = fd_submit->in_fence_fd,
182 .ring_idx = virtio_pipe->ring_idx,
183 };
184 vdrm_execbuf(to_virtio_device(dev)->vdrm, &p);
185
186 out_fence->fence_fd = p.fence_fd;
187
188 free(req);
189
190 if (!bos_on_stack) {
191 free(submit_bos);
192 free(guest_handles);
193 }
194
195 if (fd_submit->in_fence_fd != -1)
196 close(fd_submit->in_fence_fd);
197
198 fd_submit_ref(&fd_submit->base);
199
200 util_queue_fence_init(&fd_submit->retire_fence);
201
202 util_queue_add_job(&virtio_pipe->retire_queue,
203 fd_submit, &fd_submit->retire_fence,
204 retire_execute,
205 retire_cleanup,
206 0);
207
208 return 0;
209 }
210
211 struct fd_submit *
virtio_submit_new(struct fd_pipe * pipe)212 virtio_submit_new(struct fd_pipe *pipe)
213 {
214 /* We don't do any translation from internal FD_RELOC flags to MSM flags. */
215 STATIC_ASSERT(FD_RELOC_READ == MSM_SUBMIT_BO_READ);
216 STATIC_ASSERT(FD_RELOC_WRITE == MSM_SUBMIT_BO_WRITE);
217 STATIC_ASSERT(FD_RELOC_DUMP == MSM_SUBMIT_BO_DUMP);
218
219 return fd_submit_sp_new(pipe, flush_submit_list);
220 }
221