1 /*
2 * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #include "nvk_queue.h"
6
7 #include "nvk_buffer.h"
8 #include "nvk_cmd_buffer.h"
9 #include "nvk_device.h"
10 #include "nvk_image.h"
11 #include "nvk_physical_device.h"
12 #include "nv_push.h"
13
14 #include "nv_push_cl9039.h"
15 #include "nv_push_cl9097.h"
16 #include "nv_push_cl90b5.h"
17 #include "nv_push_cla0c0.h"
18 #include "cla1c0.h"
19 #include "nv_push_clc3c0.h"
20 #include "nv_push_clc397.h"
21
22 static void
nvk_queue_state_init(struct nvk_queue_state * qs)23 nvk_queue_state_init(struct nvk_queue_state *qs)
24 {
25 memset(qs, 0, sizeof(*qs));
26 }
27
28 static void
nvk_queue_state_finish(struct nvk_device * dev,struct nvk_queue_state * qs)29 nvk_queue_state_finish(struct nvk_device *dev,
30 struct nvk_queue_state *qs)
31 {
32 if (qs->images.mem)
33 nvkmd_mem_unref(qs->images.mem);
34 if (qs->samplers.mem)
35 nvkmd_mem_unref(qs->samplers.mem);
36 if (qs->slm.mem)
37 nvkmd_mem_unref(qs->slm.mem);
38 if (qs->push.mem)
39 nvkmd_mem_unref(qs->push.mem);
40 }
41
42 static void
nvk_queue_state_dump_push(struct nvk_device * dev,struct nvk_queue_state * qs,FILE * fp)43 nvk_queue_state_dump_push(struct nvk_device *dev,
44 struct nvk_queue_state *qs, FILE *fp)
45 {
46 struct nvk_physical_device *pdev = nvk_device_physical(dev);
47
48 struct nv_push push = {
49 .start = (uint32_t *)qs->push.mem->map,
50 .end = (uint32_t *)qs->push.mem->map + qs->push.dw_count,
51 };
52 vk_push_print(fp, &push, &pdev->info);
53 }
54
55 VkResult
nvk_queue_state_update(struct nvk_device * dev,struct nvk_queue_state * qs)56 nvk_queue_state_update(struct nvk_device *dev,
57 struct nvk_queue_state *qs)
58 {
59 struct nvk_physical_device *pdev = nvk_device_physical(dev);
60 struct nvkmd_mem *mem;
61 uint32_t alloc_count, bytes_per_warp, bytes_per_tpc;
62 bool dirty = false;
63
64 mem = nvk_descriptor_table_get_mem_ref(&dev->images, &alloc_count);
65 if (qs->images.mem != mem || qs->images.alloc_count != alloc_count) {
66 if (qs->images.mem)
67 nvkmd_mem_unref(qs->images.mem);
68 qs->images.mem = mem;
69 qs->images.alloc_count = alloc_count;
70 dirty = true;
71 } else {
72 /* No change */
73 if (mem)
74 nvkmd_mem_unref(mem);
75 }
76
77 mem = nvk_descriptor_table_get_mem_ref(&dev->samplers, &alloc_count);
78 if (qs->samplers.mem != mem || qs->samplers.alloc_count != alloc_count) {
79 if (qs->samplers.mem)
80 nvkmd_mem_unref(qs->samplers.mem);
81 qs->samplers.mem = mem;
82 qs->samplers.alloc_count = alloc_count;
83 dirty = true;
84 } else {
85 /* No change */
86 if (mem)
87 nvkmd_mem_unref(mem);
88 }
89
90 mem = nvk_slm_area_get_mem_ref(&dev->slm, &bytes_per_warp, &bytes_per_tpc);
91 if (qs->slm.mem != mem || qs->slm.bytes_per_warp != bytes_per_warp ||
92 qs->slm.bytes_per_tpc != bytes_per_tpc) {
93 if (qs->slm.mem)
94 nvkmd_mem_unref(qs->slm.mem);
95 qs->slm.mem = mem;
96 qs->slm.bytes_per_warp = bytes_per_warp;
97 qs->slm.bytes_per_tpc = bytes_per_tpc;
98 dirty = true;
99 } else {
100 /* No change */
101 if (mem)
102 nvkmd_mem_unref(mem);
103 }
104
105 /* TODO: We're currently depending on kernel reference counting to protect
106 * us here. If we ever stop reference counting in the kernel, we will
107 * either need to delay destruction or hold on to our extra BO references
108 * and insert a GPU stall here if anything has changed before dropping our
109 * old references.
110 */
111
112 if (!dirty)
113 return VK_SUCCESS;
114
115 struct nvkmd_mem *push_mem;
116 VkResult result = nvkmd_dev_alloc_mapped_mem(dev->nvkmd, &dev->vk.base,
117 256 * 4, 0, NVKMD_MEM_LOCAL,
118 NVKMD_MEM_MAP_WR, &push_mem);
119 if (result != VK_SUCCESS)
120 return result;
121
122 struct nv_push push;
123 nv_push_init(&push, push_mem->map, 256);
124 struct nv_push *p = &push;
125
126 if (qs->images.mem) {
127 /* Compute */
128 P_MTHD(p, NVA0C0, SET_TEX_HEADER_POOL_A);
129 P_NVA0C0_SET_TEX_HEADER_POOL_A(p, qs->images.mem->va->addr >> 32);
130 P_NVA0C0_SET_TEX_HEADER_POOL_B(p, qs->images.mem->va->addr);
131 P_NVA0C0_SET_TEX_HEADER_POOL_C(p, qs->images.alloc_count - 1);
132 P_IMMD(p, NVA0C0, INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI, {
133 .lines = LINES_ALL
134 });
135
136 /* 3D */
137 P_MTHD(p, NV9097, SET_TEX_HEADER_POOL_A);
138 P_NV9097_SET_TEX_HEADER_POOL_A(p, qs->images.mem->va->addr >> 32);
139 P_NV9097_SET_TEX_HEADER_POOL_B(p, qs->images.mem->va->addr);
140 P_NV9097_SET_TEX_HEADER_POOL_C(p, qs->images.alloc_count - 1);
141 P_IMMD(p, NV9097, INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI, {
142 .lines = LINES_ALL
143 });
144 }
145
146 if (qs->samplers.mem) {
147 /* Compute */
148 P_MTHD(p, NVA0C0, SET_TEX_SAMPLER_POOL_A);
149 P_NVA0C0_SET_TEX_SAMPLER_POOL_A(p, qs->samplers.mem->va->addr >> 32);
150 P_NVA0C0_SET_TEX_SAMPLER_POOL_B(p, qs->samplers.mem->va->addr);
151 P_NVA0C0_SET_TEX_SAMPLER_POOL_C(p, qs->samplers.alloc_count - 1);
152 P_IMMD(p, NVA0C0, INVALIDATE_SAMPLER_CACHE_NO_WFI, {
153 .lines = LINES_ALL
154 });
155
156 /* 3D */
157 P_MTHD(p, NV9097, SET_TEX_SAMPLER_POOL_A);
158 P_NV9097_SET_TEX_SAMPLER_POOL_A(p, qs->samplers.mem->va->addr >> 32);
159 P_NV9097_SET_TEX_SAMPLER_POOL_B(p, qs->samplers.mem->va->addr);
160 P_NV9097_SET_TEX_SAMPLER_POOL_C(p, qs->samplers.alloc_count - 1);
161 P_IMMD(p, NV9097, INVALIDATE_SAMPLER_CACHE_NO_WFI, {
162 .lines = LINES_ALL
163 });
164 }
165
166 if (qs->slm.mem) {
167 const uint64_t slm_addr = qs->slm.mem->va->addr;
168 const uint64_t slm_size = qs->slm.mem->size_B;
169 const uint64_t slm_per_warp = qs->slm.bytes_per_warp;
170 const uint64_t slm_per_tpc = qs->slm.bytes_per_tpc;
171 assert(!(slm_per_tpc & 0x7fff));
172
173 /* Compute */
174 P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_A);
175 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_A(p, slm_addr >> 32);
176 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_B(p, slm_addr);
177
178 P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A);
179 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A(p, slm_per_tpc >> 32);
180 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_B(p, slm_per_tpc);
181 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C(p, 0xff);
182
183 if (pdev->info.cls_compute < VOLTA_COMPUTE_A) {
184 P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_THROTTLED_A);
185 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_THROTTLED_A(p, slm_per_tpc >> 32);
186 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_THROTTLED_B(p, slm_per_tpc);
187 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_THROTTLED_C(p, 0xff);
188 }
189
190 /* 3D */
191 P_MTHD(p, NV9097, SET_SHADER_LOCAL_MEMORY_A);
192 P_NV9097_SET_SHADER_LOCAL_MEMORY_A(p, slm_addr >> 32);
193 P_NV9097_SET_SHADER_LOCAL_MEMORY_B(p, slm_addr);
194 P_NV9097_SET_SHADER_LOCAL_MEMORY_C(p, slm_size >> 32);
195 P_NV9097_SET_SHADER_LOCAL_MEMORY_D(p, slm_size);
196 P_NV9097_SET_SHADER_LOCAL_MEMORY_E(p, slm_per_warp);
197 }
198
199 /* We set memory windows unconditionally. Otherwise, the memory window
200 * might be in a random place and cause us to fault off into nowhere.
201 */
202 if (pdev->info.cls_compute >= VOLTA_COMPUTE_A) {
203 uint64_t temp = 0xfeULL << 24;
204 P_MTHD(p, NVC3C0, SET_SHADER_SHARED_MEMORY_WINDOW_A);
205 P_NVC3C0_SET_SHADER_SHARED_MEMORY_WINDOW_A(p, temp >> 32);
206 P_NVC3C0_SET_SHADER_SHARED_MEMORY_WINDOW_B(p, temp & 0xffffffff);
207
208 temp = 0xffULL << 24;
209 P_MTHD(p, NVC3C0, SET_SHADER_LOCAL_MEMORY_WINDOW_A);
210 P_NVC3C0_SET_SHADER_LOCAL_MEMORY_WINDOW_A(p, temp >> 32);
211 P_NVC3C0_SET_SHADER_LOCAL_MEMORY_WINDOW_B(p, temp & 0xffffffff);
212 } else {
213 P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_WINDOW);
214 P_NVA0C0_SET_SHADER_LOCAL_MEMORY_WINDOW(p, 0xff << 24);
215
216 P_MTHD(p, NVA0C0, SET_SHADER_SHARED_MEMORY_WINDOW);
217 P_NVA0C0_SET_SHADER_SHARED_MEMORY_WINDOW(p, 0xfe << 24);
218 }
219
220 /* From nvc0_screen.c:
221 *
222 * "Reduce likelihood of collision with real buffers by placing the
223 * hole at the top of the 4G area. This will have to be dealt with
224 * for real eventually by blocking off that area from the VM."
225 *
226 * Really?!? TODO: Fix this for realz. Annoyingly, we only have a
227 * 32-bit pointer for this in 3D rather than a full 48 like we have for
228 * compute.
229 */
230 P_IMMD(p, NV9097, SET_SHADER_LOCAL_MEMORY_WINDOW, 0xff << 24);
231
232 if (qs->push.mem)
233 nvkmd_mem_unref(qs->push.mem);
234
235 qs->push.mem = push_mem;
236 qs->push.dw_count = nv_push_dw_count(&push);
237
238 return VK_SUCCESS;
239 }
240
241 static VkResult
nvk_queue_submit_bind(struct nvk_queue * queue,struct vk_queue_submit * submit)242 nvk_queue_submit_bind(struct nvk_queue *queue,
243 struct vk_queue_submit *submit)
244 {
245 VkResult result;
246
247 result = nvkmd_ctx_wait(queue->bind_ctx, &queue->vk.base,
248 submit->wait_count, submit->waits);
249 if (result != VK_SUCCESS)
250 return result;
251
252 for (uint32_t i = 0; i < submit->buffer_bind_count; i++) {
253 result = nvk_queue_buffer_bind(queue, &submit->buffer_binds[i]);
254 if (result != VK_SUCCESS)
255 return result;
256 }
257
258 for (uint32_t i = 0; i < submit->image_bind_count; i++) {
259 result = nvk_queue_image_bind(queue, &submit->image_binds[i]);
260 if (result != VK_SUCCESS)
261 return result;
262 }
263
264 for (uint32_t i = 0; i < submit->image_opaque_bind_count; i++) {
265 result = nvk_queue_image_opaque_bind(queue, &submit->image_opaque_binds[i]);
266 if (result != VK_SUCCESS)
267 return result;
268 }
269
270 result = nvkmd_ctx_signal(queue->bind_ctx, &queue->vk.base,
271 submit->signal_count, submit->signals);
272 if (result != VK_SUCCESS)
273 return result;
274
275 return VK_SUCCESS;
276 }
277
278 static VkResult
nvk_queue_submit_exec(struct nvk_queue * queue,struct vk_queue_submit * submit)279 nvk_queue_submit_exec(struct nvk_queue *queue,
280 struct vk_queue_submit *submit)
281 {
282 struct nvk_device *dev = nvk_queue_device(queue);
283 struct nvk_physical_device *pdev = nvk_device_physical(dev);
284 VkResult result;
285
286 const bool sync = pdev->debug_flags & NVK_DEBUG_PUSH_SYNC;
287
288 uint64_t upload_time_point;
289 result = nvk_upload_queue_flush(dev, &dev->upload, &upload_time_point);
290 if (result != VK_SUCCESS)
291 return result;
292
293 if (upload_time_point > 0) {
294 struct vk_sync_wait wait = {
295 .sync = dev->upload.sync,
296 .stage_mask = ~0,
297 .wait_value = upload_time_point,
298 };
299 result = nvkmd_ctx_wait(queue->exec_ctx, &queue->vk.base, 1, &wait);
300 if (result != VK_SUCCESS)
301 goto fail;
302 }
303
304 result = nvkmd_ctx_wait(queue->exec_ctx, &queue->vk.base,
305 submit->wait_count, submit->waits);
306 if (result != VK_SUCCESS)
307 goto fail;
308
309 if (queue->state.push.mem != NULL) {
310 struct nvkmd_ctx_exec exec = {
311 .addr = queue->state.push.mem->va->addr,
312 .size_B = queue->state.push.dw_count * 4,
313 };
314 result = nvkmd_ctx_exec(queue->exec_ctx, &queue->vk.base, 1, &exec);
315 if (result != VK_SUCCESS)
316 goto fail;
317 }
318
319 for (unsigned i = 0; i < submit->command_buffer_count; i++) {
320 struct nvk_cmd_buffer *cmd =
321 container_of(submit->command_buffers[i], struct nvk_cmd_buffer, vk);
322
323 const uint32_t max_execs =
324 util_dynarray_num_elements(&cmd->pushes, struct nvk_cmd_push);
325 STACK_ARRAY(struct nvkmd_ctx_exec, execs, max_execs);
326 uint32_t exec_count = 0;
327
328 util_dynarray_foreach(&cmd->pushes, struct nvk_cmd_push, push) {
329 if (push->range == 0)
330 continue;
331
332 execs[exec_count++] = (struct nvkmd_ctx_exec) {
333 .addr = push->addr,
334 .size_B = push->range,
335 .no_prefetch = push->no_prefetch,
336 };
337 }
338
339 result = nvkmd_ctx_exec(queue->exec_ctx, &queue->vk.base,
340 exec_count, execs);
341
342 STACK_ARRAY_FINISH(execs);
343
344 if (result != VK_SUCCESS)
345 goto fail;
346 }
347
348 result = nvkmd_ctx_signal(queue->exec_ctx, &queue->vk.base,
349 submit->signal_count, submit->signals);
350 if (result != VK_SUCCESS)
351 goto fail;
352
353 if (sync) {
354 result = nvkmd_ctx_sync(queue->exec_ctx, &queue->vk.base);
355 if (result != VK_SUCCESS)
356 goto fail;
357 }
358
359 fail:
360 if ((sync && result != VK_SUCCESS) ||
361 (pdev->debug_flags & NVK_DEBUG_PUSH_DUMP)) {
362 nvk_queue_state_dump_push(dev, &queue->state, stderr);
363
364 for (unsigned i = 0; i < submit->command_buffer_count; i++) {
365 struct nvk_cmd_buffer *cmd =
366 container_of(submit->command_buffers[i], struct nvk_cmd_buffer, vk);
367
368 nvk_cmd_buffer_dump(cmd, stderr);
369 }
370 }
371
372 return result;
373 }
374
375 static VkResult
nvk_queue_submit(struct vk_queue * vk_queue,struct vk_queue_submit * submit)376 nvk_queue_submit(struct vk_queue *vk_queue,
377 struct vk_queue_submit *submit)
378 {
379 struct nvk_queue *queue = container_of(vk_queue, struct nvk_queue, vk);
380 struct nvk_device *dev = nvk_queue_device(queue);
381 VkResult result;
382
383 if (vk_queue_is_lost(&queue->vk))
384 return VK_ERROR_DEVICE_LOST;
385
386 if (submit->buffer_bind_count > 0 ||
387 submit->image_bind_count > 0 ||
388 submit->image_opaque_bind_count > 0) {
389 assert(submit->command_buffer_count == 0);
390 result = nvk_queue_submit_bind(queue, submit);
391 if (result != VK_SUCCESS)
392 return vk_queue_set_lost(&queue->vk, "Bind operation failed");
393 } else {
394 result = nvk_queue_state_update(dev, &queue->state);
395 if (result != VK_SUCCESS) {
396 return vk_queue_set_lost(&queue->vk, "Failed to update queue base "
397 "pointers pushbuf");
398 }
399
400 result = nvk_queue_submit_exec(queue, submit);
401 if (result != VK_SUCCESS)
402 return vk_queue_set_lost(&queue->vk, "Submit failed");
403 }
404
405 return VK_SUCCESS;
406 }
407
408 static VkResult
nvk_queue_submit_simple(struct nvk_queue * queue,uint32_t dw_count,const uint32_t * dw)409 nvk_queue_submit_simple(struct nvk_queue *queue,
410 uint32_t dw_count, const uint32_t *dw)
411 {
412 struct nvk_device *dev = nvk_queue_device(queue);
413 struct nvk_physical_device *pdev = nvk_device_physical(dev);
414 VkResult result;
415
416 if (vk_queue_is_lost(&queue->vk))
417 return VK_ERROR_DEVICE_LOST;
418
419 struct nvkmd_mem *push_mem;
420 result = nvkmd_dev_alloc_mapped_mem(dev->nvkmd, &dev->vk.base,
421 dw_count * 4, 0,
422 NVKMD_MEM_GART,
423 NVKMD_MEM_MAP_WR, &push_mem);
424 if (result != VK_SUCCESS)
425 return result;
426
427 memcpy(push_mem->map, dw, dw_count * 4);
428
429 const struct nvkmd_ctx_exec exec = {
430 .addr = push_mem->va->addr,
431 .size_B = dw_count * 4,
432 };
433 result = nvkmd_ctx_exec(queue->exec_ctx, &queue->vk.base, 1, &exec);
434 if (result == VK_SUCCESS)
435 result = nvkmd_ctx_sync(queue->exec_ctx, &queue->vk.base);
436
437 nvkmd_mem_unref(push_mem);
438
439 const bool debug_sync = pdev->debug_flags & NVK_DEBUG_PUSH_SYNC;
440 if ((debug_sync && result != VK_SUCCESS) ||
441 (pdev->debug_flags & NVK_DEBUG_PUSH_DUMP)) {
442 struct nv_push push = {
443 .start = (uint32_t *)dw,
444 .end = (uint32_t *)dw + dw_count,
445 };
446 vk_push_print(stderr, &push, &pdev->info);
447 }
448
449 if (result != VK_SUCCESS)
450 return vk_queue_set_lost(&queue->vk, "Submit failed");
451
452 return VK_SUCCESS;
453 }
454
455 static VkResult
nvk_queue_init_context_state(struct nvk_queue * queue,enum nvkmd_engines engines)456 nvk_queue_init_context_state(struct nvk_queue *queue,
457 enum nvkmd_engines engines)
458 {
459 struct nvk_device *dev = nvk_queue_device(queue);
460 struct nvk_physical_device *pdev = nvk_device_physical(dev);
461 VkResult result;
462
463 uint32_t push_data[4096];
464 struct nv_push push;
465 nv_push_init(&push, push_data, ARRAY_SIZE(push_data));
466 struct nv_push *p = &push;
467
468 /* M2MF state */
469 if (pdev->info.cls_m2mf <= FERMI_MEMORY_TO_MEMORY_FORMAT_A) {
470 /* we absolutely do not support Fermi, but if somebody wants to toy
471 * around with it, this is a must
472 */
473 P_MTHD(p, NV9039, SET_OBJECT);
474 P_NV9039_SET_OBJECT(p, {
475 .class_id = pdev->info.cls_m2mf,
476 .engine_id = 0,
477 });
478 }
479
480 if (engines & NVKMD_ENGINE_3D) {
481 result = nvk_push_draw_state_init(queue, p);
482 if (result != VK_SUCCESS)
483 return result;
484 }
485
486 if (engines & NVKMD_ENGINE_COMPUTE) {
487 result = nvk_push_dispatch_state_init(queue, p);
488 if (result != VK_SUCCESS)
489 return result;
490 }
491
492 return nvk_queue_submit_simple(queue, nv_push_dw_count(&push), push_data);
493 }
494
495 VkResult
nvk_queue_init(struct nvk_device * dev,struct nvk_queue * queue,const VkDeviceQueueCreateInfo * pCreateInfo,uint32_t index_in_family)496 nvk_queue_init(struct nvk_device *dev, struct nvk_queue *queue,
497 const VkDeviceQueueCreateInfo *pCreateInfo,
498 uint32_t index_in_family)
499 {
500 struct nvk_physical_device *pdev = nvk_device_physical(dev);
501 VkResult result;
502
503 assert(pCreateInfo->queueFamilyIndex < pdev->queue_family_count);
504 const struct nvk_queue_family *queue_family =
505 &pdev->queue_families[pCreateInfo->queueFamilyIndex];
506
507 result = vk_queue_init(&queue->vk, &dev->vk, pCreateInfo, index_in_family);
508 if (result != VK_SUCCESS)
509 return result;
510
511 nvk_queue_state_init(&queue->state);
512
513 enum nvkmd_engines engines = 0;
514 if (queue_family->queue_flags & VK_QUEUE_GRAPHICS_BIT) {
515 engines |= NVKMD_ENGINE_3D;
516 /* We rely on compute shaders for queries */
517 engines |= NVKMD_ENGINE_COMPUTE;
518 }
519 if (queue_family->queue_flags & VK_QUEUE_COMPUTE_BIT) {
520 engines |= NVKMD_ENGINE_COMPUTE;
521 /* We currently rely on 3D engine MMEs for indirect dispatch */
522 engines |= NVKMD_ENGINE_3D;
523 }
524 if (queue_family->queue_flags & VK_QUEUE_TRANSFER_BIT)
525 engines |= NVKMD_ENGINE_COPY;
526
527 if (engines) {
528 result = nvkmd_dev_create_ctx(dev->nvkmd, &dev->vk.base,
529 engines, &queue->exec_ctx);
530 if (result != VK_SUCCESS)
531 goto fail_init;
532
533 result = nvkmd_dev_alloc_mem(dev->nvkmd, &dev->vk.base,
534 4096, 0, NVKMD_MEM_LOCAL,
535 &queue->draw_cb0);
536 if (result != VK_SUCCESS)
537 goto fail_exec_ctx;
538
539 result = nvk_upload_queue_fill(dev, &dev->upload,
540 queue->draw_cb0->va->addr, 0,
541 queue->draw_cb0->size_B);
542 if (result != VK_SUCCESS)
543 goto fail_draw_cb0;
544 }
545
546 if (queue_family->queue_flags & VK_QUEUE_SPARSE_BINDING_BIT) {
547 result = nvkmd_dev_create_ctx(dev->nvkmd, &dev->vk.base,
548 NVKMD_ENGINE_BIND, &queue->bind_ctx);
549 if (result != VK_SUCCESS)
550 goto fail_draw_cb0;
551 }
552
553 result = nvk_queue_init_context_state(queue, engines);
554 if (result != VK_SUCCESS)
555 goto fail_bind_ctx;
556
557 queue->vk.driver_submit = nvk_queue_submit;
558
559 return VK_SUCCESS;
560
561 fail_bind_ctx:
562 if (queue->bind_ctx != NULL)
563 nvkmd_ctx_destroy(queue->bind_ctx);
564 fail_draw_cb0:
565 if (queue->draw_cb0 != NULL)
566 nvkmd_mem_unref(queue->draw_cb0);
567 fail_exec_ctx:
568 if (queue->exec_ctx != NULL)
569 nvkmd_ctx_destroy(queue->exec_ctx);
570 fail_init:
571 nvk_queue_state_finish(dev, &queue->state);
572 vk_queue_finish(&queue->vk);
573
574 return result;
575 }
576
577 void
nvk_queue_finish(struct nvk_device * dev,struct nvk_queue * queue)578 nvk_queue_finish(struct nvk_device *dev, struct nvk_queue *queue)
579 {
580 if (queue->draw_cb0 != NULL) {
581 nvk_upload_queue_sync(dev, &dev->upload);
582 nvkmd_mem_unref(queue->draw_cb0);
583 }
584 nvk_queue_state_finish(dev, &queue->state);
585 if (queue->bind_ctx != NULL)
586 nvkmd_ctx_destroy(queue->bind_ctx);
587 if (queue->exec_ctx != NULL)
588 nvkmd_ctx_destroy(queue->exec_ctx);
589 vk_queue_finish(&queue->vk);
590 }
591