1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <errno.h>
26 #include <fcntl.h>
27 #include <limits.h>
28 #include <stdbool.h>
29 #include <stddef.h>
30 #include <stdint.h>
31 #include <string.h>
32 #include <unistd.h>
33 #include <vulkan/vulkan.h>
34
35 #include "fw-api/pvr_rogue_fwif.h"
36 #include "fw-api/pvr_rogue_fwif_rf.h"
37 #include "pvr_csb.h"
38 #include "pvr_job_render.h"
39 #include "pvr_private.h"
40 #include "pvr_srv.h"
41 #include "pvr_srv_bo.h"
42 #include "pvr_srv_bridge.h"
43 #include "pvr_srv_job_common.h"
44 #include "pvr_srv_job_render.h"
45 #include "pvr_srv_sync.h"
46 #include "pvr_srv_sync_prim.h"
47 #include "pvr_types.h"
48 #include "pvr_winsys.h"
49 #include "util/log.h"
50 #include "util/macros.h"
51 #include "util/u_math.h"
52 #include "vk_alloc.h"
53 #include "vk_log.h"
54 #include "vk_util.h"
55
56 struct pvr_srv_winsys_free_list {
57 struct pvr_winsys_free_list base;
58
59 void *handle;
60
61 struct pvr_srv_winsys_free_list *parent;
62 };
63
64 #define to_pvr_srv_winsys_free_list(free_list) \
65 container_of(free_list, struct pvr_srv_winsys_free_list, base)
66
67 struct pvr_srv_winsys_rt_dataset {
68 struct pvr_winsys_rt_dataset base;
69
70 struct {
71 void *handle;
72 struct pvr_srv_sync_prim *sync_prim;
73 } rt_datas[ROGUE_FWIF_NUM_RTDATAS];
74 };
75
76 #define to_pvr_srv_winsys_rt_dataset(rt_dataset) \
77 container_of(rt_dataset, struct pvr_srv_winsys_rt_dataset, base)
78
79 struct pvr_srv_winsys_render_ctx {
80 struct pvr_winsys_render_ctx base;
81
82 /* Handle to kernel context. */
83 void *handle;
84
85 int timeline_geom;
86 int timeline_frag;
87 };
88
89 #define to_pvr_srv_winsys_render_ctx(ctx) \
90 container_of(ctx, struct pvr_srv_winsys_render_ctx, base)
91
pvr_srv_winsys_free_list_create(struct pvr_winsys * ws,struct pvr_winsys_vma * free_list_vma,uint32_t initial_num_pages,uint32_t max_num_pages,uint32_t grow_num_pages,uint32_t grow_threshold,struct pvr_winsys_free_list * parent_free_list,struct pvr_winsys_free_list ** const free_list_out)92 VkResult pvr_srv_winsys_free_list_create(
93 struct pvr_winsys *ws,
94 struct pvr_winsys_vma *free_list_vma,
95 uint32_t initial_num_pages,
96 uint32_t max_num_pages,
97 uint32_t grow_num_pages,
98 uint32_t grow_threshold,
99 struct pvr_winsys_free_list *parent_free_list,
100 struct pvr_winsys_free_list **const free_list_out)
101 {
102 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws);
103 struct pvr_srv_winsys_bo *srv_free_list_bo =
104 to_pvr_srv_winsys_bo(free_list_vma->bo);
105 struct pvr_srv_winsys_free_list *srv_free_list;
106 void *parent_handle;
107 VkResult result;
108
109 srv_free_list = vk_zalloc(ws->alloc,
110 sizeof(*srv_free_list),
111 8,
112 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
113 if (!srv_free_list)
114 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
115
116 if (parent_free_list) {
117 srv_free_list->parent = to_pvr_srv_winsys_free_list(parent_free_list);
118 parent_handle = srv_free_list->parent->handle;
119 } else {
120 srv_free_list->parent = NULL;
121 parent_handle = NULL;
122 }
123
124 result = pvr_srv_rgx_create_free_list(ws->render_fd,
125 srv_ws->server_memctx_data,
126 max_num_pages,
127 initial_num_pages,
128 grow_num_pages,
129 grow_threshold,
130 parent_handle,
131 #if MESA_DEBUG
132 PVR_SRV_TRUE /* free_list_check */,
133 #else
134 PVR_SRV_FALSE /* free_list_check */,
135 #endif
136 free_list_vma->dev_addr,
137 srv_free_list_bo->pmr,
138 0 /* pmr_offset */,
139 &srv_free_list->handle);
140 if (result != VK_SUCCESS)
141 goto err_vk_free_srv_free_list;
142
143 srv_free_list->base.ws = ws;
144
145 *free_list_out = &srv_free_list->base;
146
147 return VK_SUCCESS;
148
149 err_vk_free_srv_free_list:
150 vk_free(ws->alloc, srv_free_list);
151
152 return result;
153 }
154
pvr_srv_winsys_free_list_destroy(struct pvr_winsys_free_list * free_list)155 void pvr_srv_winsys_free_list_destroy(struct pvr_winsys_free_list *free_list)
156 {
157 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(free_list->ws);
158 struct pvr_srv_winsys_free_list *srv_free_list =
159 to_pvr_srv_winsys_free_list(free_list);
160
161 pvr_srv_rgx_destroy_free_list(srv_ws->base.render_fd, srv_free_list->handle);
162 vk_free(srv_ws->base.alloc, srv_free_list);
163 }
164
pvr_rogue_get_cr_multisamplectl_val(uint32_t samples,bool y_flip)165 static uint64_t pvr_rogue_get_cr_multisamplectl_val(uint32_t samples,
166 bool y_flip)
167 {
168 static const struct {
169 uint8_t x[8];
170 uint8_t y[8];
171 } sample_positions[4] = {
172 /* 1 sample */
173 {
174 .x = { 8 },
175 .y = { 8 },
176 },
177 /* 2 samples */
178 {
179 .x = { 12, 4 },
180 .y = { 12, 4 },
181 },
182 /* 4 samples */
183 {
184 .x = { 6, 14, 2, 10 },
185 .y = { 2, 6, 10, 14 },
186 },
187 /* 8 samples */
188 {
189 .x = { 9, 7, 13, 5, 3, 1, 11, 15 },
190 .y = { 5, 11, 9, 3, 13, 7, 15, 1 },
191 },
192 };
193 uint64_t multisamplectl;
194 uint8_t idx;
195
196 idx = util_fast_log2(samples);
197 assert(idx < ARRAY_SIZE(sample_positions));
198
199 pvr_csb_pack (&multisamplectl, CR_PPP_MULTISAMPLECTL, value) {
200 switch (samples) {
201 case 8:
202 value.msaa_x7 = sample_positions[idx].x[7];
203 value.msaa_x6 = sample_positions[idx].x[6];
204 value.msaa_x5 = sample_positions[idx].x[5];
205 value.msaa_x4 = sample_positions[idx].x[4];
206
207 if (y_flip) {
208 value.msaa_y7 = 16U - sample_positions[idx].y[7];
209 value.msaa_y6 = 16U - sample_positions[idx].y[6];
210 value.msaa_y5 = 16U - sample_positions[idx].y[5];
211 value.msaa_y4 = 16U - sample_positions[idx].y[4];
212 } else {
213 value.msaa_y7 = sample_positions[idx].y[7];
214 value.msaa_y6 = sample_positions[idx].y[6];
215 value.msaa_y5 = sample_positions[idx].y[5];
216 value.msaa_y4 = sample_positions[idx].y[4];
217 }
218
219 FALLTHROUGH;
220 case 4:
221 value.msaa_x3 = sample_positions[idx].x[3];
222 value.msaa_x2 = sample_positions[idx].x[2];
223
224 if (y_flip) {
225 value.msaa_y3 = 16U - sample_positions[idx].y[3];
226 value.msaa_y2 = 16U - sample_positions[idx].y[2];
227 } else {
228 value.msaa_y3 = sample_positions[idx].y[3];
229 value.msaa_y2 = sample_positions[idx].y[2];
230 }
231
232 FALLTHROUGH;
233 case 2:
234 value.msaa_x1 = sample_positions[idx].x[1];
235
236 if (y_flip) {
237 value.msaa_y1 = 16U - sample_positions[idx].y[1];
238 } else {
239 value.msaa_y1 = sample_positions[idx].y[1];
240 }
241
242 FALLTHROUGH;
243 case 1:
244 value.msaa_x0 = sample_positions[idx].x[0];
245
246 if (y_flip) {
247 value.msaa_y0 = 16U - sample_positions[idx].y[0];
248 } else {
249 value.msaa_y0 = sample_positions[idx].y[0];
250 }
251
252 break;
253 default:
254 unreachable("Unsupported number of samples");
255 }
256 }
257
258 return multisamplectl;
259 }
260
261 static uint32_t
pvr_rogue_get_cr_isp_mtile_size_val(const struct pvr_device_info * dev_info,const struct pvr_rt_mtile_info * mtile_info,uint32_t samples)262 pvr_rogue_get_cr_isp_mtile_size_val(const struct pvr_device_info *dev_info,
263 const struct pvr_rt_mtile_info *mtile_info,
264 uint32_t samples)
265 {
266 uint32_t samples_per_pixel =
267 PVR_GET_FEATURE_VALUE(dev_info, isp_samples_per_pixel, 0);
268 uint32_t isp_mtile_size;
269
270 pvr_csb_pack (&isp_mtile_size, CR_ISP_MTILE_SIZE, value) {
271 value.x = mtile_info->mtile_x1;
272 value.y = mtile_info->mtile_y1;
273
274 if (samples_per_pixel == 1) {
275 if (samples >= 4)
276 value.x <<= 1;
277
278 if (samples >= 2)
279 value.y <<= 1;
280 } else if (samples_per_pixel == 2) {
281 if (samples >= 8)
282 value.x <<= 1;
283
284 if (samples >= 4)
285 value.y <<= 1;
286 } else if (samples_per_pixel == 4) {
287 if (samples >= 8)
288 value.y <<= 1;
289 } else {
290 assert(!"Unsupported ISP samples per pixel value");
291 }
292 }
293
294 return isp_mtile_size;
295 }
296
pvr_rogue_get_ppp_screen_val(uint32_t width,uint32_t height)297 static uint32_t pvr_rogue_get_ppp_screen_val(uint32_t width, uint32_t height)
298 {
299 uint32_t val;
300
301 pvr_csb_pack (&val, CR_PPP_SCREEN, state) {
302 state.pixxmax = width - 1;
303 state.pixymax = height - 1;
304 }
305
306 return val;
307 }
308
309 struct pvr_rogue_cr_te {
310 uint32_t aa;
311 uint32_t mtile1;
312 uint32_t mtile2;
313 uint32_t screen;
314 uint32_t mtile_stride;
315 };
316
pvr_rogue_ct_te_init(const struct pvr_device_info * dev_info,const struct pvr_rt_mtile_info * mtile_info,uint32_t samples,struct pvr_rogue_cr_te * const te_regs)317 static void pvr_rogue_ct_te_init(const struct pvr_device_info *dev_info,
318 const struct pvr_rt_mtile_info *mtile_info,
319 uint32_t samples,
320 struct pvr_rogue_cr_te *const te_regs)
321 {
322 uint32_t samples_per_pixel =
323 PVR_GET_FEATURE_VALUE(dev_info, isp_samples_per_pixel, 0);
324
325 pvr_csb_pack (&te_regs->aa, CR_TE_AA, value) {
326 if (samples_per_pixel == 1) {
327 if (samples >= 2)
328 value.y = true;
329 if (samples >= 4)
330 value.x = true;
331 } else if (samples_per_pixel == 2) {
332 if (samples >= 2)
333 value.x2 = true;
334 if (samples >= 4)
335 value.y = true;
336 if (samples >= 8)
337 value.x = true;
338 } else if (samples_per_pixel == 4) {
339 if (samples >= 2)
340 value.x2 = true;
341 if (samples >= 4)
342 value.y2 = true;
343 if (samples >= 8)
344 value.y = true;
345 } else {
346 assert(!"Unsupported ISP samples per pixel value");
347 }
348 }
349
350 pvr_csb_pack (&te_regs->mtile1, CR_TE_MTILE1, value) {
351 value.x1 = mtile_info->mtile_x1;
352 if (!PVR_HAS_FEATURE(dev_info, simple_internal_parameter_format)) {
353 value.x2 = mtile_info->mtile_x2;
354 value.x3 = mtile_info->mtile_x3;
355 }
356 }
357
358 pvr_csb_pack (&te_regs->mtile2, CR_TE_MTILE2, value) {
359 value.y1 = mtile_info->mtile_y1;
360 if (!PVR_HAS_FEATURE(dev_info, simple_internal_parameter_format)) {
361 value.y2 = mtile_info->mtile_y2;
362 value.y3 = mtile_info->mtile_y3;
363 }
364 }
365
366 pvr_csb_pack (&te_regs->screen, CR_TE_SCREEN, value) {
367 value.xmax = mtile_info->x_tile_max;
368 value.ymax = mtile_info->y_tile_max;
369 }
370
371 te_regs->mtile_stride = mtile_info->mtile_x1 * mtile_info->mtile_y1;
372 }
373
pvr_srv_render_target_dataset_create(struct pvr_winsys * ws,const struct pvr_winsys_rt_dataset_create_info * create_info,const struct pvr_device_info * dev_info,struct pvr_winsys_rt_dataset ** const rt_dataset_out)374 VkResult pvr_srv_render_target_dataset_create(
375 struct pvr_winsys *ws,
376 const struct pvr_winsys_rt_dataset_create_info *create_info,
377 const struct pvr_device_info *dev_info,
378 struct pvr_winsys_rt_dataset **const rt_dataset_out)
379 {
380 const pvr_dev_addr_t macrotile_addrs[ROGUE_FWIF_NUM_RTDATAS] = {
381 [0] = create_info->rt_datas[0].macrotile_array_dev_addr,
382 [1] = create_info->rt_datas[1].macrotile_array_dev_addr,
383 };
384 const pvr_dev_addr_t pm_mlist_addrs[ROGUE_FWIF_NUM_RTDATAS] = {
385 [0] = create_info->rt_datas[0].pm_mlist_dev_addr,
386 [1] = create_info->rt_datas[1].pm_mlist_dev_addr,
387 };
388 const pvr_dev_addr_t rgn_header_addrs[ROGUE_FWIF_NUM_RTDATAS] = {
389 [0] = create_info->rt_datas[0].rgn_header_dev_addr,
390 [1] = create_info->rt_datas[1].rgn_header_dev_addr,
391 };
392
393 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws);
394 struct pvr_srv_winsys_free_list *srv_local_free_list =
395 to_pvr_srv_winsys_free_list(create_info->local_free_list);
396 void *free_lists[ROGUE_FW_MAX_FREELISTS] = { NULL };
397 struct pvr_srv_winsys_rt_dataset *srv_rt_dataset;
398 void *handles[ROGUE_FWIF_NUM_RTDATAS];
399 struct pvr_rogue_cr_te rogue_te_regs;
400 struct pvr_rt_mtile_info mtile_info;
401 uint32_t isp_mtile_size;
402 VkResult result;
403
404 free_lists[ROGUE_FW_LOCAL_FREELIST] = srv_local_free_list->handle;
405
406 if (srv_local_free_list->parent) {
407 free_lists[ROGUE_FW_GLOBAL_FREELIST] =
408 srv_local_free_list->parent->handle;
409 }
410
411 srv_rt_dataset = vk_zalloc(ws->alloc,
412 sizeof(*srv_rt_dataset),
413 8,
414 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
415 if (!srv_rt_dataset)
416 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
417
418 /* If greater than 1 we'll have to pass in an array. For now just passing in
419 * the reference.
420 */
421 STATIC_ASSERT(ROGUE_FWIF_NUM_GEOMDATAS == 1);
422 /* If not 2 the arrays used in the bridge call will require updating. */
423 STATIC_ASSERT(ROGUE_FWIF_NUM_RTDATAS == 2);
424
425 pvr_rt_mtile_info_init(dev_info,
426 &mtile_info,
427 create_info->width,
428 create_info->height,
429 create_info->samples);
430
431 isp_mtile_size = pvr_rogue_get_cr_isp_mtile_size_val(dev_info,
432 &mtile_info,
433 create_info->samples);
434
435 pvr_rogue_ct_te_init(dev_info,
436 &mtile_info,
437 create_info->samples,
438 &rogue_te_regs);
439
440 result = pvr_srv_rgx_create_hwrt_dataset(
441 ws->render_fd,
442 pvr_rogue_get_cr_multisamplectl_val(create_info->samples, true),
443 pvr_rogue_get_cr_multisamplectl_val(create_info->samples, false),
444 macrotile_addrs,
445 pm_mlist_addrs,
446 &create_info->rtc_dev_addr,
447 rgn_header_addrs,
448 &create_info->tpc_dev_addr,
449 &create_info->vheap_table_dev_addr,
450 free_lists,
451 create_info->isp_merge_lower_x,
452 create_info->isp_merge_lower_y,
453 create_info->isp_merge_scale_x,
454 create_info->isp_merge_scale_y,
455 create_info->isp_merge_upper_x,
456 create_info->isp_merge_upper_y,
457 isp_mtile_size,
458 rogue_te_regs.mtile_stride,
459 pvr_rogue_get_ppp_screen_val(create_info->width, create_info->height),
460 create_info->rgn_header_size,
461 rogue_te_regs.aa,
462 rogue_te_regs.mtile1,
463 rogue_te_regs.mtile2,
464 rogue_te_regs.screen,
465 create_info->tpc_size,
466 create_info->tpc_stride,
467 create_info->layers,
468 handles);
469 if (result != VK_SUCCESS)
470 goto err_vk_free_srv_rt_dataset;
471
472 srv_rt_dataset->rt_datas[0].handle = handles[0];
473 srv_rt_dataset->rt_datas[1].handle = handles[1];
474
475 for (uint32_t i = 0; i < ARRAY_SIZE(srv_rt_dataset->rt_datas); i++) {
476 srv_rt_dataset->rt_datas[i].sync_prim = pvr_srv_sync_prim_alloc(srv_ws);
477 if (!srv_rt_dataset->rt_datas[i].sync_prim)
478 goto err_srv_sync_prim_free;
479 }
480
481 srv_rt_dataset->base.ws = ws;
482
483 *rt_dataset_out = &srv_rt_dataset->base;
484
485 return VK_SUCCESS;
486
487 err_srv_sync_prim_free:
488 for (uint32_t i = 0; i < ARRAY_SIZE(srv_rt_dataset->rt_datas); i++) {
489 pvr_srv_sync_prim_free(srv_ws, srv_rt_dataset->rt_datas[i].sync_prim);
490
491 if (srv_rt_dataset->rt_datas[i].handle) {
492 pvr_srv_rgx_destroy_hwrt_dataset(ws->render_fd,
493 srv_rt_dataset->rt_datas[i].handle);
494 }
495 }
496
497 err_vk_free_srv_rt_dataset:
498 vk_free(ws->alloc, srv_rt_dataset);
499
500 return result;
501 }
502
pvr_srv_render_target_dataset_destroy(struct pvr_winsys_rt_dataset * rt_dataset)503 void pvr_srv_render_target_dataset_destroy(
504 struct pvr_winsys_rt_dataset *rt_dataset)
505 {
506 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(rt_dataset->ws);
507 struct pvr_srv_winsys_rt_dataset *srv_rt_dataset =
508 to_pvr_srv_winsys_rt_dataset(rt_dataset);
509
510 for (uint32_t i = 0; i < ARRAY_SIZE(srv_rt_dataset->rt_datas); i++) {
511 pvr_srv_sync_prim_free(srv_ws, srv_rt_dataset->rt_datas[i].sync_prim);
512
513 if (srv_rt_dataset->rt_datas[i].handle) {
514 pvr_srv_rgx_destroy_hwrt_dataset(srv_ws->base.render_fd,
515 srv_rt_dataset->rt_datas[i].handle);
516 }
517 }
518
519 vk_free(srv_ws->base.alloc, srv_rt_dataset);
520 }
521
pvr_srv_render_ctx_fw_static_state_init(struct pvr_winsys_render_ctx_create_info * create_info,struct rogue_fwif_static_rendercontext_state * static_state)522 static void pvr_srv_render_ctx_fw_static_state_init(
523 struct pvr_winsys_render_ctx_create_info *create_info,
524 struct rogue_fwif_static_rendercontext_state *static_state)
525 {
526 struct pvr_winsys_render_ctx_static_state *ws_static_state =
527 &create_info->static_state;
528 struct rogue_fwif_ta_regs_cswitch *regs =
529 &static_state->ctx_switch_geom_regs[0];
530
531 memset(static_state, 0, sizeof(*static_state));
532
533 regs->vdm_context_state_base_addr = ws_static_state->vdm_ctx_state_base_addr;
534 regs->ta_context_state_base_addr = ws_static_state->geom_ctx_state_base_addr;
535
536 STATIC_ASSERT(ARRAY_SIZE(regs->ta_state) ==
537 ARRAY_SIZE(ws_static_state->geom_state));
538 for (uint32_t i = 0; i < ARRAY_SIZE(ws_static_state->geom_state); i++) {
539 regs->ta_state[i].vdm_context_store_task0 =
540 ws_static_state->geom_state[i].vdm_ctx_store_task0;
541 regs->ta_state[i].vdm_context_store_task1 =
542 ws_static_state->geom_state[i].vdm_ctx_store_task1;
543 regs->ta_state[i].vdm_context_store_task2 =
544 ws_static_state->geom_state[i].vdm_ctx_store_task2;
545
546 regs->ta_state[i].vdm_context_resume_task0 =
547 ws_static_state->geom_state[i].vdm_ctx_resume_task0;
548 regs->ta_state[i].vdm_context_resume_task1 =
549 ws_static_state->geom_state[i].vdm_ctx_resume_task1;
550 regs->ta_state[i].vdm_context_resume_task2 =
551 ws_static_state->geom_state[i].vdm_ctx_resume_task2;
552 }
553 }
554
pvr_srv_winsys_render_ctx_create(struct pvr_winsys * ws,struct pvr_winsys_render_ctx_create_info * create_info,struct pvr_winsys_render_ctx ** const ctx_out)555 VkResult pvr_srv_winsys_render_ctx_create(
556 struct pvr_winsys *ws,
557 struct pvr_winsys_render_ctx_create_info *create_info,
558 struct pvr_winsys_render_ctx **const ctx_out)
559 {
560 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ws);
561 struct rogue_fwif_rf_cmd reset_cmd = { 0 };
562
563 struct rogue_fwif_static_rendercontext_state static_state;
564 struct pvr_srv_winsys_render_ctx *srv_ctx;
565 const uint32_t call_stack_depth = 1U;
566 VkResult result;
567
568 srv_ctx = vk_zalloc(ws->alloc,
569 sizeof(*srv_ctx),
570 8,
571 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
572 if (!srv_ctx)
573 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
574
575 result = pvr_srv_create_timeline(ws->render_fd, &srv_ctx->timeline_geom);
576 if (result != VK_SUCCESS)
577 goto err_free_srv_ctx;
578
579 result = pvr_srv_create_timeline(ws->render_fd, &srv_ctx->timeline_frag);
580 if (result != VK_SUCCESS)
581 goto err_close_timeline_geom;
582
583 pvr_srv_render_ctx_fw_static_state_init(create_info, &static_state);
584
585 /* TODO: Add support for reset framework. Currently we subtract
586 * reset_cmd.regs size from reset_cmd size to only pass empty flags field.
587 */
588 result = pvr_srv_rgx_create_render_context(
589 ws->render_fd,
590 pvr_srv_from_winsys_priority(create_info->priority),
591 create_info->vdm_callstack_addr,
592 call_stack_depth,
593 sizeof(reset_cmd) - sizeof(reset_cmd.regs),
594 (uint8_t *)&reset_cmd,
595 srv_ws->server_memctx_data,
596 sizeof(static_state),
597 (uint8_t *)&static_state,
598 0,
599 RGX_CONTEXT_FLAG_DISABLESLR,
600 0,
601 UINT_MAX,
602 UINT_MAX,
603 &srv_ctx->handle);
604 if (result != VK_SUCCESS)
605 goto err_close_timeline_frag;
606
607 srv_ctx->base.ws = ws;
608
609 *ctx_out = &srv_ctx->base;
610
611 return VK_SUCCESS;
612
613 err_close_timeline_frag:
614 close(srv_ctx->timeline_frag);
615
616 err_close_timeline_geom:
617 close(srv_ctx->timeline_geom);
618
619 err_free_srv_ctx:
620 vk_free(ws->alloc, srv_ctx);
621
622 return vk_error(NULL, VK_ERROR_INITIALIZATION_FAILED);
623 }
624
pvr_srv_winsys_render_ctx_destroy(struct pvr_winsys_render_ctx * ctx)625 void pvr_srv_winsys_render_ctx_destroy(struct pvr_winsys_render_ctx *ctx)
626 {
627 struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ctx->ws);
628 struct pvr_srv_winsys_render_ctx *srv_ctx =
629 to_pvr_srv_winsys_render_ctx(ctx);
630
631 pvr_srv_rgx_destroy_render_context(srv_ws->base.render_fd, srv_ctx->handle);
632 close(srv_ctx->timeline_frag);
633 close(srv_ctx->timeline_geom);
634 vk_free(srv_ws->base.alloc, srv_ctx);
635 }
636
637 static uint32_t
pvr_srv_geometry_cmd_stream_load(struct rogue_fwif_cmd_ta * const cmd,const uint8_t * const stream,const uint32_t stream_len,const struct pvr_device_info * const dev_info)638 pvr_srv_geometry_cmd_stream_load(struct rogue_fwif_cmd_ta *const cmd,
639 const uint8_t *const stream,
640 const uint32_t stream_len,
641 const struct pvr_device_info *const dev_info)
642 {
643 const uint32_t *stream_ptr = (const uint32_t *)stream;
644 struct rogue_fwif_ta_regs *const regs = &cmd->regs;
645 uint32_t main_stream_len =
646 pvr_csb_unpack((const uint64_t *)stream_ptr, KMD_STREAM_HDR).length;
647
648 stream_ptr += pvr_cmd_length(KMD_STREAM_HDR);
649
650 regs->vdm_ctrl_stream_base = *(const uint64_t *)stream_ptr;
651 stream_ptr += pvr_cmd_length(CR_VDM_CTRL_STREAM_BASE);
652
653 regs->tpu_border_colour_table = *(const uint64_t *)stream_ptr;
654 stream_ptr += pvr_cmd_length(CR_TPU_BORDER_COLOUR_TABLE_VDM);
655
656 regs->ppp_ctrl = *stream_ptr;
657 stream_ptr += pvr_cmd_length(CR_PPP_CTRL);
658
659 regs->te_psg = *stream_ptr;
660 stream_ptr += pvr_cmd_length(CR_TE_PSG);
661
662 regs->vdm_context_resume_task0_size = *stream_ptr;
663 stream_ptr += pvr_cmd_length(VDMCTRL_PDS_STATE0);
664
665 regs->view_idx = *stream_ptr;
666 stream_ptr++;
667
668 assert((const uint8_t *)stream_ptr - stream <= stream_len);
669 assert((const uint8_t *)stream_ptr - stream == main_stream_len);
670
671 return main_stream_len;
672 }
673
pvr_srv_geometry_cmd_ext_stream_load(struct rogue_fwif_cmd_ta * const cmd,const uint8_t * const stream,const uint32_t stream_len,const uint32_t ext_stream_offset,const struct pvr_device_info * const dev_info)674 static void pvr_srv_geometry_cmd_ext_stream_load(
675 struct rogue_fwif_cmd_ta *const cmd,
676 const uint8_t *const stream,
677 const uint32_t stream_len,
678 const uint32_t ext_stream_offset,
679 const struct pvr_device_info *const dev_info)
680 {
681 const uint32_t *ext_stream_ptr =
682 (const uint32_t *)((uint8_t *)stream + ext_stream_offset);
683 struct rogue_fwif_ta_regs *const regs = &cmd->regs;
684
685 struct PVRX(KMD_STREAM_EXTHDR_GEOM0) header0;
686
687 header0 = pvr_csb_unpack(ext_stream_ptr, KMD_STREAM_EXTHDR_GEOM0);
688 ext_stream_ptr += pvr_cmd_length(KMD_STREAM_EXTHDR_GEOM0);
689
690 assert(PVR_HAS_QUIRK(dev_info, 49927) == header0.has_brn49927);
691 if (header0.has_brn49927) {
692 regs->tpu = *ext_stream_ptr;
693 ext_stream_ptr += pvr_cmd_length(CR_TPU);
694 }
695
696 assert((const uint8_t *)ext_stream_ptr - stream == stream_len);
697 }
698
pvr_srv_geometry_cmd_init(const struct pvr_winsys_render_submit_info * submit_info,const struct pvr_srv_sync_prim * sync_prim,struct rogue_fwif_cmd_ta * cmd,const struct pvr_device_info * const dev_info)699 static void pvr_srv_geometry_cmd_init(
700 const struct pvr_winsys_render_submit_info *submit_info,
701 const struct pvr_srv_sync_prim *sync_prim,
702 struct rogue_fwif_cmd_ta *cmd,
703 const struct pvr_device_info *const dev_info)
704 {
705 const struct pvr_winsys_geometry_state *state = &submit_info->geometry;
706 uint32_t ext_stream_offset;
707
708 memset(cmd, 0, sizeof(*cmd));
709
710 cmd->cmd_shared.cmn.frame_num = submit_info->frame_num;
711
712 ext_stream_offset = pvr_srv_geometry_cmd_stream_load(cmd,
713 state->fw_stream,
714 state->fw_stream_len,
715 dev_info);
716
717 if (ext_stream_offset < state->fw_stream_len) {
718 pvr_srv_geometry_cmd_ext_stream_load(cmd,
719 state->fw_stream,
720 state->fw_stream_len,
721 ext_stream_offset,
722 dev_info);
723 }
724
725 if (state->flags.is_first_geometry)
726 cmd->flags |= ROGUE_FWIF_TAFLAGS_FIRSTKICK;
727
728 if (state->flags.is_last_geometry)
729 cmd->flags |= ROGUE_FWIF_TAFLAGS_LASTKICK;
730
731 if (state->flags.use_single_core)
732 cmd->flags |= ROGUE_FWIF_TAFLAGS_SINGLE_CORE;
733
734 cmd->partial_render_ta_3d_fence.ufo_addr.addr =
735 pvr_srv_sync_prim_get_fw_addr(sync_prim);
736 cmd->partial_render_ta_3d_fence.value = sync_prim->value;
737 }
738
739 static uint32_t
pvr_srv_fragment_cmd_stream_load(struct rogue_fwif_cmd_3d * const cmd,const uint8_t * const stream,const uint32_t stream_len,const struct pvr_device_info * const dev_info)740 pvr_srv_fragment_cmd_stream_load(struct rogue_fwif_cmd_3d *const cmd,
741 const uint8_t *const stream,
742 const uint32_t stream_len,
743 const struct pvr_device_info *const dev_info)
744 {
745 const uint32_t *stream_ptr = (const uint32_t *)stream;
746 struct rogue_fwif_3d_regs *const regs = &cmd->regs;
747 uint32_t main_stream_len =
748 pvr_csb_unpack((const uint64_t *)stream_ptr, KMD_STREAM_HDR).length;
749
750 stream_ptr += pvr_cmd_length(KMD_STREAM_HDR);
751
752 regs->isp_scissor_base = *(const uint64_t *)stream_ptr;
753 stream_ptr += pvr_cmd_length(CR_ISP_SCISSOR_BASE);
754
755 regs->isp_dbias_base = *(const uint64_t *)stream_ptr;
756 stream_ptr += pvr_cmd_length(CR_ISP_DBIAS_BASE);
757
758 regs->isp_oclqry_base = *(const uint64_t *)stream_ptr;
759 stream_ptr += pvr_cmd_length(CR_ISP_OCLQRY_BASE);
760
761 regs->isp_zlsctl = *(const uint64_t *)stream_ptr;
762 stream_ptr += pvr_cmd_length(CR_ISP_ZLSCTL);
763
764 regs->isp_zload_store_base = *(const uint64_t *)stream_ptr;
765 stream_ptr += pvr_cmd_length(CR_ISP_ZLOAD_BASE);
766
767 regs->isp_stencil_load_store_base = *(const uint64_t *)stream_ptr;
768 stream_ptr += pvr_cmd_length(CR_ISP_STENCIL_LOAD_BASE);
769
770 if (PVR_HAS_FEATURE(dev_info, requires_fb_cdc_zls_setup)) {
771 regs->fb_cdc_zls = *(const uint64_t *)stream_ptr;
772 stream_ptr += 2U;
773 }
774
775 STATIC_ASSERT(ARRAY_SIZE(regs->pbe_word) == 8U);
776 STATIC_ASSERT(ARRAY_SIZE(regs->pbe_word[0]) == 3U);
777 STATIC_ASSERT(sizeof(regs->pbe_word[0][0]) == sizeof(uint64_t));
778 memcpy(regs->pbe_word, stream_ptr, sizeof(regs->pbe_word));
779 stream_ptr += 8U * 3U * 2U;
780
781 regs->tpu_border_colour_table = *(const uint64_t *)stream_ptr;
782 stream_ptr += pvr_cmd_length(CR_TPU_BORDER_COLOUR_TABLE_PDM);
783
784 STATIC_ASSERT(ARRAY_SIZE(regs->pds_bgnd) == 3U);
785 STATIC_ASSERT(sizeof(regs->pds_bgnd[0]) == sizeof(uint64_t));
786 memcpy(regs->pds_bgnd, stream_ptr, sizeof(regs->pds_bgnd));
787 stream_ptr += 3U * 2U;
788
789 STATIC_ASSERT(ARRAY_SIZE(regs->pds_pr_bgnd) == 3U);
790 STATIC_ASSERT(sizeof(regs->pds_pr_bgnd[0]) == sizeof(uint64_t));
791 memcpy(regs->pds_pr_bgnd, stream_ptr, sizeof(regs->pds_pr_bgnd));
792 stream_ptr += 3U * 2U;
793
794 STATIC_ASSERT(ARRAY_SIZE(regs->usc_clear_register) == 8U);
795 STATIC_ASSERT(sizeof(regs->usc_clear_register[0]) == sizeof(uint32_t));
796 memcpy(regs->usc_clear_register,
797 stream_ptr,
798 sizeof(regs->usc_clear_register));
799 stream_ptr += 8U;
800
801 regs->usc_pixel_output_ctrl = *stream_ptr;
802 stream_ptr += pvr_cmd_length(CR_USC_PIXEL_OUTPUT_CTRL);
803
804 regs->isp_bgobjdepth = *stream_ptr;
805 stream_ptr += pvr_cmd_length(CR_ISP_BGOBJDEPTH);
806
807 regs->isp_bgobjvals = *stream_ptr;
808 stream_ptr += pvr_cmd_length(CR_ISP_BGOBJVALS);
809
810 regs->isp_aa = *stream_ptr;
811 stream_ptr += pvr_cmd_length(CR_ISP_AA);
812
813 regs->isp_ctl = *stream_ptr;
814 stream_ptr += pvr_cmd_length(CR_ISP_CTL);
815
816 regs->event_pixel_pds_info = *stream_ptr;
817 stream_ptr += pvr_cmd_length(CR_EVENT_PIXEL_PDS_INFO);
818
819 if (PVR_HAS_FEATURE(dev_info, cluster_grouping)) {
820 regs->pixel_phantom = *stream_ptr;
821 stream_ptr++;
822 }
823
824 regs->view_idx = *stream_ptr;
825 stream_ptr++;
826
827 regs->event_pixel_pds_data = *stream_ptr;
828 stream_ptr += pvr_cmd_length(CR_EVENT_PIXEL_PDS_DATA);
829
830 if (PVR_HAS_FEATURE(dev_info, gpu_multicore_support)) {
831 regs->isp_oclqry_stride = *stream_ptr;
832 stream_ptr++;
833 }
834
835 if (PVR_HAS_FEATURE(dev_info, zls_subtile)) {
836 regs->isp_zls_pixels = *stream_ptr;
837 stream_ptr += pvr_cmd_length(CR_ISP_ZLS_PIXELS);
838 }
839
840 cmd->zls_stride = *stream_ptr;
841 stream_ptr++;
842
843 cmd->sls_stride = *stream_ptr;
844 stream_ptr++;
845
846 if (PVR_HAS_FEATURE(dev_info, gpu_multicore_support)) {
847 cmd->execute_count = *stream_ptr;
848 stream_ptr++;
849 }
850
851 assert((const uint8_t *)stream_ptr - stream <= stream_len);
852 assert((const uint8_t *)stream_ptr - stream == main_stream_len);
853
854 return main_stream_len;
855 }
856
pvr_srv_fragment_cmd_ext_stream_load(struct rogue_fwif_cmd_3d * const cmd,const uint8_t * const stream,const uint32_t stream_len,const uint32_t ext_stream_offset,const struct pvr_device_info * const dev_info)857 static void pvr_srv_fragment_cmd_ext_stream_load(
858 struct rogue_fwif_cmd_3d *const cmd,
859 const uint8_t *const stream,
860 const uint32_t stream_len,
861 const uint32_t ext_stream_offset,
862 const struct pvr_device_info *const dev_info)
863 {
864 const uint32_t *ext_stream_ptr =
865 (const uint32_t *)((uint8_t *)stream + ext_stream_offset);
866 struct rogue_fwif_3d_regs *const regs = &cmd->regs;
867
868 struct PVRX(KMD_STREAM_EXTHDR_FRAG0) header0;
869
870 header0 = pvr_csb_unpack(ext_stream_ptr, KMD_STREAM_EXTHDR_FRAG0);
871 ext_stream_ptr += pvr_cmd_length(KMD_STREAM_EXTHDR_FRAG0);
872
873 assert(PVR_HAS_QUIRK(dev_info, 49927) == header0.has_brn49927);
874 if (header0.has_brn49927) {
875 regs->tpu = *ext_stream_ptr;
876 ext_stream_ptr += pvr_cmd_length(CR_TPU);
877 }
878
879 assert((const uint8_t *)ext_stream_ptr - stream == stream_len);
880 }
881
882 static void
pvr_srv_fragment_cmd_init(struct rogue_fwif_cmd_3d * cmd,const struct pvr_winsys_fragment_state * state,const struct pvr_device_info * dev_info,uint32_t frame_num)883 pvr_srv_fragment_cmd_init(struct rogue_fwif_cmd_3d *cmd,
884 const struct pvr_winsys_fragment_state *state,
885 const struct pvr_device_info *dev_info,
886 uint32_t frame_num)
887 {
888 uint32_t ext_stream_offset;
889
890 memset(cmd, 0, sizeof(*cmd));
891
892 cmd->cmd_shared.cmn.frame_num = frame_num;
893
894 ext_stream_offset = pvr_srv_fragment_cmd_stream_load(cmd,
895 state->fw_stream,
896 state->fw_stream_len,
897 dev_info);
898
899 if (ext_stream_offset < state->fw_stream_len) {
900 pvr_srv_fragment_cmd_ext_stream_load(cmd,
901 state->fw_stream,
902 state->fw_stream_len,
903 ext_stream_offset,
904 dev_info);
905 }
906
907 if (state->flags.has_depth_buffer)
908 cmd->flags |= ROGUE_FWIF_RENDERFLAGS_DEPTHBUFFER;
909
910 if (state->flags.has_stencil_buffer)
911 cmd->flags |= ROGUE_FWIF_RENDERFLAGS_STENCILBUFFER;
912
913 if (state->flags.prevent_cdm_overlap)
914 cmd->flags |= ROGUE_FWIF_RENDERFLAGS_PREVENT_CDM_OVERLAP;
915
916 if (state->flags.use_single_core)
917 cmd->flags |= ROGUE_FWIF_RENDERFLAGS_SINGLE_CORE;
918
919 if (state->flags.get_vis_results)
920 cmd->flags |= ROGUE_FWIF_RENDERFLAGS_GETVISRESULTS;
921
922 if (state->flags.has_spm_scratch_buffer)
923 cmd->flags |= ROGUE_FWIF_RENDERFLAGS_SPMSCRATCHBUFFER;
924 }
925
pvr_srv_winsys_render_submit(const struct pvr_winsys_render_ctx * ctx,const struct pvr_winsys_render_submit_info * submit_info,const struct pvr_device_info * dev_info,struct vk_sync * signal_sync_geom,struct vk_sync * signal_sync_frag)926 VkResult pvr_srv_winsys_render_submit(
927 const struct pvr_winsys_render_ctx *ctx,
928 const struct pvr_winsys_render_submit_info *submit_info,
929 const struct pvr_device_info *dev_info,
930 struct vk_sync *signal_sync_geom,
931 struct vk_sync *signal_sync_frag)
932 {
933 const struct pvr_srv_winsys_rt_dataset *srv_rt_dataset =
934 to_pvr_srv_winsys_rt_dataset(submit_info->rt_dataset);
935 struct pvr_srv_sync_prim *sync_prim =
936 srv_rt_dataset->rt_datas[submit_info->rt_data_idx].sync_prim;
937 void *rt_data_handle =
938 srv_rt_dataset->rt_datas[submit_info->rt_data_idx].handle;
939 const struct pvr_srv_winsys_render_ctx *srv_ctx =
940 to_pvr_srv_winsys_render_ctx(ctx);
941 const struct pvr_srv_winsys *srv_ws = to_pvr_srv_winsys(ctx->ws);
942
943 struct pvr_srv_sync *srv_signal_sync_geom;
944 struct pvr_srv_sync *srv_signal_sync_frag;
945
946 struct rogue_fwif_cmd_ta geom_cmd;
947 struct rogue_fwif_cmd_3d frag_cmd = { 0 };
948 struct rogue_fwif_cmd_3d pr_cmd = { 0 };
949
950 uint8_t *frag_cmd_ptr = NULL;
951 uint32_t frag_cmd_size = 0;
952
953 uint32_t current_sync_value = sync_prim->value;
954 uint32_t geom_sync_update_value;
955 uint32_t frag_to_geom_fence_count = 0;
956 uint32_t frag_to_geom_fence_value;
957 uint32_t frag_sync_update_count = 0;
958 uint32_t frag_sync_update_value;
959
960 int in_frag_fd = -1;
961 int in_geom_fd = -1;
962 int fence_frag;
963 int fence_geom;
964
965 VkResult result;
966
967 pvr_srv_geometry_cmd_init(submit_info, sync_prim, &geom_cmd, dev_info);
968
969 pvr_srv_fragment_cmd_init(&pr_cmd,
970 &submit_info->fragment_pr,
971 dev_info,
972 submit_info->frame_num);
973
974 if (submit_info->has_fragment_job) {
975 pvr_srv_fragment_cmd_init(&frag_cmd,
976 &submit_info->fragment,
977 dev_info,
978 submit_info->frame_num);
979
980 frag_cmd_ptr = (uint8_t *)&frag_cmd;
981 frag_cmd_size = sizeof(frag_cmd);
982 }
983
984 if (submit_info->geometry.wait) {
985 struct pvr_srv_sync *srv_wait_sync =
986 to_srv_sync(submit_info->geometry.wait);
987
988 if (srv_wait_sync->fd >= 0) {
989 in_geom_fd = dup(srv_wait_sync->fd);
990 if (in_geom_fd == -1) {
991 return vk_errorf(NULL,
992 VK_ERROR_OUT_OF_HOST_MEMORY,
993 "dup called on wait sync failed, Errno: %s",
994 strerror(errno));
995 }
996 }
997 }
998
999 if (submit_info->fragment.wait) {
1000 struct pvr_srv_sync *srv_wait_sync =
1001 to_srv_sync(submit_info->fragment.wait);
1002
1003 if (srv_wait_sync->fd >= 0) {
1004 in_frag_fd = dup(srv_wait_sync->fd);
1005 if (in_frag_fd == -1) {
1006 return vk_errorf(NULL,
1007 VK_ERROR_OUT_OF_HOST_MEMORY,
1008 "dup called on wait sync failed, Errno: %s",
1009 strerror(errno));
1010 }
1011 }
1012 }
1013
1014 if (submit_info->geometry.flags.is_first_geometry) {
1015 frag_to_geom_fence_count = 1;
1016 frag_to_geom_fence_value = current_sync_value;
1017 }
1018
1019 /* Geometery is always kicked */
1020 geom_sync_update_value = ++current_sync_value;
1021
1022 if (submit_info->has_fragment_job) {
1023 frag_sync_update_count = 1;
1024 frag_sync_update_value = ++current_sync_value;
1025 }
1026
1027 do {
1028 /* The fw allows the ZS and MSAA scratch buffers to be lazily allocated in
1029 * which case we need to provide a status update (i.e. if they are
1030 * physically backed or not) to the fw. In our case they will always be
1031 * physically backed so no need to inform the fw about their status and
1032 * pass in anything. We'll just pass in NULL.
1033 */
1034 result = pvr_srv_rgx_kick_render2(srv_ws->base.render_fd,
1035 srv_ctx->handle,
1036 frag_to_geom_fence_count,
1037 &sync_prim->ctx->block_handle,
1038 &sync_prim->offset,
1039 &frag_to_geom_fence_value,
1040 1,
1041 &sync_prim->ctx->block_handle,
1042 &sync_prim->offset,
1043 &geom_sync_update_value,
1044 frag_sync_update_count,
1045 &sync_prim->ctx->block_handle,
1046 &sync_prim->offset,
1047 &frag_sync_update_value,
1048 sync_prim->ctx->block_handle,
1049 sync_prim->offset,
1050 geom_sync_update_value,
1051 in_geom_fd,
1052 srv_ctx->timeline_geom,
1053 &fence_geom,
1054 "GEOM",
1055 in_frag_fd,
1056 srv_ctx->timeline_frag,
1057 &fence_frag,
1058 "FRAG",
1059 sizeof(geom_cmd),
1060 (uint8_t *)&geom_cmd,
1061 sizeof(pr_cmd),
1062 (uint8_t *)&pr_cmd,
1063 frag_cmd_size,
1064 frag_cmd_ptr,
1065 submit_info->job_num,
1066 /* Always kick the TA. */
1067 true,
1068 /* Always kick a PR. */
1069 true,
1070 submit_info->has_fragment_job,
1071 false,
1072 0,
1073 rt_data_handle,
1074 NULL,
1075 NULL,
1076 0,
1077 NULL,
1078 NULL,
1079 0,
1080 0,
1081 0,
1082 0,
1083 0);
1084 } while (result == VK_NOT_READY);
1085
1086 if (result != VK_SUCCESS)
1087 goto end_close_in_fds;
1088
1089 /* The job submission was succesful, update the sync prim value. */
1090 sync_prim->value = current_sync_value;
1091
1092 if (signal_sync_geom) {
1093 srv_signal_sync_geom = to_srv_sync(signal_sync_geom);
1094 pvr_srv_set_sync_payload(srv_signal_sync_geom, fence_geom);
1095 } else if (fence_geom != -1) {
1096 close(fence_geom);
1097 }
1098
1099 if (signal_sync_frag) {
1100 srv_signal_sync_frag = to_srv_sync(signal_sync_frag);
1101 pvr_srv_set_sync_payload(srv_signal_sync_frag, fence_frag);
1102 } else if (fence_frag != -1) {
1103 close(fence_frag);
1104 }
1105
1106 end_close_in_fds:
1107 if (in_geom_fd >= 0)
1108 close(in_geom_fd);
1109
1110 if (in_frag_fd >= 0)
1111 close(in_frag_fd);
1112
1113 return result;
1114 }
1115