xref: /aosp_15_r20/external/mesa3d/src/gallium/drivers/d3d12/d3d12_copy.cpp (revision 6104692788411f58d303aa86923a9ff6ecaded22) 
1 /*
2  * Copyright © Microsoft Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #include "d3d12_context.h"
25 #include "d3d12_compiler.h"
26 #include "d3d12_debug.h"
27 #include "d3d12_format.h"
28 #include "d3d12_query.h"
29 #include "d3d12_resource.h"
30 #include "d3d12_screen.h"
31 
32 #include "util/u_blitter.h"
33 #include "util/format/u_format.h"
34 
35 #include "nir_to_dxil.h"
36 #include "nir_builder.h"
37 
38 static void
copy_buffer_region_no_barriers(struct d3d12_context * ctx,struct d3d12_resource * dst,uint64_t dst_offset,struct d3d12_resource * src,uint64_t src_offset,uint64_t size)39 copy_buffer_region_no_barriers(struct d3d12_context *ctx,
40                                struct d3d12_resource *dst,
41                                uint64_t dst_offset,
42                                struct d3d12_resource *src,
43                                uint64_t src_offset,
44                                uint64_t size)
45 {
46    uint64_t dst_off, src_off;
47    ID3D12Resource *dst_buf = d3d12_resource_underlying(dst, &dst_off);
48    ID3D12Resource *src_buf = d3d12_resource_underlying(src, &src_off);
49 
50    ctx->cmdlist->CopyBufferRegion(dst_buf, dst_offset + dst_off,
51                                   src_buf, src_offset + src_off,
52                                   size);
53 }
54 
55 inline static unsigned
get_subresource_id(enum pipe_texture_target target,unsigned subres,unsigned stride,unsigned z,unsigned * updated_z,unsigned array_size,unsigned plane_slice)56 get_subresource_id(enum pipe_texture_target target, unsigned subres, unsigned stride,
57                    unsigned z, unsigned *updated_z, unsigned array_size, unsigned plane_slice)
58 {
59    if (d3d12_subresource_id_uses_layer(target)) {
60       subres += stride * z;
61       if (updated_z)
62          *updated_z = 0;
63    }
64    return subres + plane_slice * array_size * stride;
65 }
66 
67 static void
copy_subregion_no_barriers(struct d3d12_context * ctx,struct d3d12_resource * dst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct d3d12_resource * src,unsigned src_level,const struct pipe_box * psrc_box,unsigned mask)68 copy_subregion_no_barriers(struct d3d12_context *ctx,
69                            struct d3d12_resource *dst,
70                            unsigned dst_level,
71                            unsigned dstx, unsigned dsty, unsigned dstz,
72                            struct d3d12_resource *src,
73                            unsigned src_level,
74                            const struct pipe_box *psrc_box,
75                            unsigned mask)
76 {
77    UNUSED struct d3d12_screen *screen = d3d12_screen(ctx->base.screen);
78    D3D12_TEXTURE_COPY_LOCATION src_loc, dst_loc;
79    unsigned src_z = psrc_box->z;
80 
81    int src_subres_stride = src->base.b.last_level + 1;
82    int dst_subres_stride = dst->base.b.last_level + 1;
83 
84    int src_array_size = src->base.b.array_size;
85    int dst_array_size = dst->base.b.array_size;
86 
87    int stencil_src_res_offset = 1;
88    int stencil_dst_res_offset = 1;
89 
90    int src_nres = 1;
91    int dst_nres = 1;
92 
93    if (dst->base.b.format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
94        dst->base.b.format == PIPE_FORMAT_S8_UINT_Z24_UNORM ||
95        dst->base.b.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
96       stencil_dst_res_offset = dst_subres_stride * dst_array_size;
97       src_nres = 2;
98    }
99 
100    if (src->base.b.format == PIPE_FORMAT_Z24_UNORM_S8_UINT ||
101        src->base.b.format == PIPE_FORMAT_S8_UINT_Z24_UNORM ||
102        dst->base.b.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) {
103       stencil_src_res_offset = src_subres_stride * src_array_size;
104       dst_nres = 2;
105    }
106 
107    static_assert(PIPE_MASK_S == 0x20 && PIPE_MASK_Z == 0x10, "unexpected ZS format mask");
108    int nsubres = MIN2(src_nres, dst_nres);
109    unsigned subresource_copy_mask = nsubres > 1 ? mask >> 4 : 1;
110 
111    for (int subres = 0; subres < nsubres; ++subres) {
112 
113       if (!(subresource_copy_mask & (1 << subres)))
114          continue;
115 
116       src_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
117       src_loc.SubresourceIndex = get_subresource_id(src->base.b.target, src_level, src_subres_stride, src_z, &src_z, src_array_size, src->plane_slice) +
118                                  subres * stencil_src_res_offset;
119       src_loc.pResource = d3d12_resource_resource(src);
120 
121       dst_loc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
122       dst_loc.SubresourceIndex = get_subresource_id(dst->base.b.target, dst_level, dst_subres_stride, dstz, &dstz, dst_array_size, dst->plane_slice) +
123                                  subres * stencil_dst_res_offset;
124       dst_loc.pResource = d3d12_resource_resource(dst);
125 
126       if (psrc_box->x == 0 && psrc_box->y == 0 && psrc_box->z == 0 &&
127           psrc_box->width == (int)u_minify(src->base.b.width0, src_level) &&
128           psrc_box->height == (int)u_minify(src->base.b.height0, src_level) &&
129           psrc_box->depth == (int)u_minify(src->base.b.depth0, src_level)) {
130 
131          assert((dstx == 0 && dsty == 0 && dstz == 0) ||
132                 screen->opts2.ProgrammableSamplePositionsTier !=
133                 D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED ||
134                 (!util_format_is_depth_or_stencil(dst->base.b.format) &&
135                  !util_format_is_depth_or_stencil(src->base.b.format) &&
136                   dst->base.b.nr_samples == src->base.b.nr_samples));
137 
138          ctx->cmdlist->CopyTextureRegion(&dst_loc, dstx, dsty, dstz,
139                                          &src_loc, NULL);
140 
141       } else {
142          D3D12_BOX src_box;
143          src_box.left = psrc_box->x;
144          src_box.right = MIN2(psrc_box->x + psrc_box->width, (int)u_minify(src->base.b.width0, src_level));
145          src_box.top = psrc_box->y;
146          src_box.bottom = MIN2(psrc_box->y + psrc_box->height, (int)u_minify(src->base.b.height0, src_level));
147          src_box.front = src_z;
148          src_box.back = src_z + psrc_box->depth;
149 
150          assert((screen->opts2.ProgrammableSamplePositionsTier !=
151                  D3D12_PROGRAMMABLE_SAMPLE_POSITIONS_TIER_NOT_SUPPORTED ||
152                  (!util_format_is_depth_or_stencil(dst->base.b.format) &&
153                   !util_format_is_depth_or_stencil(src->base.b.format))) &&
154                 dst->base.b.nr_samples == src->base.b.nr_samples);
155 
156          ctx->cmdlist->CopyTextureRegion(&dst_loc, dstx, dsty, dstz,
157                                          &src_loc, &src_box);
158       }
159    }
160 }
161 
162 static void
copy_resource_y_flipped_no_barriers(struct d3d12_context * ctx,struct d3d12_resource * dst,unsigned dst_level,const struct pipe_box * pdst_box,struct d3d12_resource * src,unsigned src_level,const struct pipe_box * psrc_box,unsigned mask)163 copy_resource_y_flipped_no_barriers(struct d3d12_context *ctx,
164                                     struct d3d12_resource *dst,
165                                     unsigned dst_level,
166                                     const struct pipe_box *pdst_box,
167                                     struct d3d12_resource *src,
168                                     unsigned src_level,
169                                     const struct pipe_box *psrc_box,
170                                     unsigned mask)
171 {
172    if (D3D12_DEBUG_BLIT & d3d12_debug) {
173       debug_printf("D3D12 BLIT as COPY: from %s@%d %dx%dx%d + %dx%dx%d\n",
174                    util_format_name(src->base.b.format), src_level,
175                    psrc_box->x, psrc_box->y, psrc_box->z,
176                    psrc_box->width, psrc_box->height, psrc_box->depth);
177       debug_printf("      to   %s@%d %dx%dx%d\n",
178                    util_format_name(dst->base.b.format), dst_level,
179                    pdst_box->x, pdst_box->y, pdst_box->z);
180    }
181 
182    struct pipe_box src_box = *psrc_box;
183    int src_inc = psrc_box->height > 0 ? 1 : -1;
184    int dst_inc = pdst_box->height > 0 ? 1 : -1;
185    src_box.height = 1;
186    int rows_to_copy = abs(psrc_box->height);
187 
188    if (psrc_box->height < 0)
189       --src_box.y;
190 
191    for (int y = 0, dest_y = pdst_box->y; y < rows_to_copy;
192         ++y, src_box.y += src_inc, dest_y += dst_inc) {
193       copy_subregion_no_barriers(ctx, dst, dst_level,
194                                  pdst_box->x, dest_y, pdst_box->z,
195                                  src, src_level, &src_box, mask);
196    }
197 }
198 
199 void
d3d12_direct_copy(struct d3d12_context * ctx,struct d3d12_resource * dst,unsigned dst_level,const struct pipe_box * pdst_box,struct d3d12_resource * src,unsigned src_level,const struct pipe_box * psrc_box,unsigned mask)200 d3d12_direct_copy(struct d3d12_context *ctx,
201                   struct d3d12_resource *dst,
202                   unsigned dst_level,
203                   const struct pipe_box *pdst_box,
204                   struct d3d12_resource *src,
205                   unsigned src_level,
206                   const struct pipe_box *psrc_box,
207                   unsigned mask)
208 {
209    struct d3d12_batch *batch = d3d12_current_batch(ctx);
210 
211    unsigned src_subres = get_subresource_id(src->base.b.target, src_level, src->base.b.last_level + 1,
212                                             psrc_box->z, nullptr, src->base.b.array_size, src->plane_slice);
213    unsigned dst_subres = get_subresource_id(dst->base.b.target, dst_level, dst->base.b.last_level + 1,
214                                             pdst_box->z, nullptr, dst->base.b.array_size, dst->plane_slice);
215 
216    if (D3D12_DEBUG_BLIT & d3d12_debug)
217       debug_printf("BLIT: Direct copy from subres %d to subres  %d\n",
218                    src_subres, dst_subres);
219 
220    d3d12_transition_subresources_state(ctx, src, src_subres, 1, 0, 1,
221                                        d3d12_get_format_start_plane(src->base.b.format),
222                                        d3d12_get_format_num_planes(src->base.b.format),
223                                        D3D12_RESOURCE_STATE_COPY_SOURCE,
224                                        D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
225 
226    d3d12_transition_subresources_state(ctx, dst, dst_subres, 1, 0, 1,
227                                        d3d12_get_format_start_plane(dst->base.b.format),
228                                        d3d12_get_format_num_planes(dst->base.b.format),
229                                        D3D12_RESOURCE_STATE_COPY_DEST,
230                                        D3D12_TRANSITION_FLAG_INVALIDATE_BINDINGS);
231 
232    d3d12_apply_resource_states(ctx, false);
233 
234    d3d12_batch_reference_resource(batch, src, false);
235    d3d12_batch_reference_resource(batch, dst, true);
236 
237    if (src->base.b.target == PIPE_BUFFER) {
238       copy_buffer_region_no_barriers(ctx, dst, pdst_box->x,
239                                      src, psrc_box->x, psrc_box->width);
240    } else if (psrc_box->height == pdst_box->height) {
241       /* No flipping, we can forward this directly to resource_copy_region */
242       copy_subregion_no_barriers(ctx, dst, dst_level,
243                                  pdst_box->x, pdst_box->y, pdst_box->z,
244                                  src, src_level, psrc_box, mask);
245    } else {
246       assert(psrc_box->height == -pdst_box->height);
247       copy_resource_y_flipped_no_barriers(ctx, dst, dst_level, pdst_box,
248                                           src, src_level, psrc_box, mask);
249    }
250 }
251 
252 struct pipe_resource *
create_staging_resource(struct d3d12_context * ctx,struct d3d12_resource * src,unsigned src_level,const struct pipe_box * src_box,struct pipe_box * dst_box,unsigned mask)253 create_staging_resource(struct d3d12_context *ctx,
254                         struct d3d12_resource *src,
255                         unsigned src_level,
256                         const struct pipe_box *src_box,
257                         struct pipe_box *dst_box,
258                         unsigned mask)
259 
260 {
261    struct pipe_resource templ = {};
262    struct pipe_resource *staging_res;
263    struct pipe_box copy_src;
264 
265    u_box_3d(MIN2(src_box->x, src_box->x + src_box->width),
266             MIN2(src_box->y, src_box->y + src_box->height),
267             MIN2(src_box->z, src_box->z + src_box->depth),
268             abs(src_box->width), abs(src_box->height), abs(src_box->depth),
269             &copy_src);
270 
271    templ.format = src->base.b.format;
272    templ.width0 = copy_src.width;
273    templ.height0 = copy_src.height;
274    templ.depth0 = copy_src.depth;
275    templ.array_size = 1;
276    templ.nr_samples = src->base.b.nr_samples;
277    templ.nr_storage_samples = src->base.b.nr_storage_samples;
278    templ.usage = PIPE_USAGE_STAGING;
279    templ.bind = util_format_is_depth_or_stencil(templ.format) ? PIPE_BIND_DEPTH_STENCIL :
280       util_format_is_compressed(templ.format) ? 0 : PIPE_BIND_RENDER_TARGET;
281    templ.target = src->base.b.target;
282 
283    staging_res = ctx->base.screen->resource_create(ctx->base.screen, &templ);
284 
285    dst_box->x = 0;
286    dst_box->y = 0;
287    dst_box->z = 0;
288    dst_box->width = copy_src.width;
289    dst_box->height = copy_src.height;
290    dst_box->depth = copy_src.depth;
291 
292    d3d12_direct_copy(ctx, d3d12_resource(staging_res), 0, dst_box,
293                      src, src_level, &copy_src, mask);
294 
295    if (src_box->width < 0) {
296       dst_box->x = dst_box->width;
297       dst_box->width = src_box->width;
298    }
299 
300    if (src_box->height < 0) {
301       dst_box->y = dst_box->height;
302       dst_box->height = src_box->height;
303    }
304 
305    if (src_box->depth < 0) {
306       dst_box->z = dst_box->depth;
307       dst_box->depth = src_box->depth;
308    }
309    return staging_res;
310 }
311 
312 static void
d3d12_resource_copy_region(struct pipe_context * pctx,struct pipe_resource * pdst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * psrc,unsigned src_level,const struct pipe_box * psrc_box)313 d3d12_resource_copy_region(struct pipe_context *pctx,
314                            struct pipe_resource *pdst,
315                            unsigned dst_level,
316                            unsigned dstx, unsigned dsty, unsigned dstz,
317                            struct pipe_resource *psrc,
318                            unsigned src_level,
319                            const struct pipe_box *psrc_box)
320 {
321    struct d3d12_context *ctx = d3d12_context(pctx);
322    struct d3d12_resource *dst = d3d12_resource(pdst);
323    struct d3d12_resource *src = d3d12_resource(psrc);
324    struct pipe_resource *staging_res = NULL;
325    const struct pipe_box *src_box = psrc_box;
326    struct pipe_box staging_box, dst_box;
327 
328    if (D3D12_DEBUG_BLIT & d3d12_debug) {
329       debug_printf("D3D12 COPY: from %s@%d msaa:%d mips:%d %dx%dx%d + %dx%dx%d\n",
330                    util_format_name(psrc->format), src_level, psrc->nr_samples,
331                    psrc->last_level,
332                    psrc_box->x, psrc_box->y, psrc_box->z,
333                    psrc_box->width, psrc_box->height, psrc_box->depth);
334       debug_printf("            to   %s@%d msaa:%d mips:%d %dx%dx%d\n",
335                    util_format_name(pdst->format), dst_level, psrc->nr_samples,
336                    psrc->last_level, dstx, dsty, dstz);
337    }
338 
339    /* Use an intermediate resource if copying from/to the same subresource */
340    if (d3d12_resource_resource(dst) == d3d12_resource_resource(src) && dst_level == src_level) {
341       staging_res = create_staging_resource(ctx, src, src_level, psrc_box, &staging_box, PIPE_MASK_RGBAZS);
342       src = d3d12_resource(staging_res);
343       src_level = 0;
344       src_box = &staging_box;
345    }
346 
347    dst_box.x = dstx;
348    dst_box.y = dsty;
349    dst_box.z = dstz;
350    dst_box.width = psrc_box->width;
351    dst_box.height = psrc_box->height;
352 
353    d3d12_direct_copy(ctx, dst, dst_level, &dst_box,
354                      src, src_level, src_box, PIPE_MASK_RGBAZS);
355 
356    if (staging_res)
357       pipe_resource_reference(&staging_res, NULL);
358 }
359 
360 void
d3d12_context_copy_init(struct pipe_context * ctx)361 d3d12_context_copy_init(struct pipe_context *ctx)
362 {
363    ctx->resource_copy_region = d3d12_resource_copy_region;
364 }
365