1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * based on si_state.c
6 * Copyright © 2015 Advanced Micro Devices, Inc.
7 *
8 * SPDX-License-Identifier: MIT
9 */
10
11 #include "radv_cp_dma.h"
12 #include "radv_buffer.h"
13 #include "radv_cs.h"
14 #include "radv_debug.h"
15 #include "radv_shader.h"
16 #include "radv_sqtt.h"
17 #include "sid.h"
18
19 /* Set this if you want the 3D engine to wait until CP DMA is done.
20 * It should be set on the last CP DMA packet. */
21 #define CP_DMA_SYNC (1 << 0)
22
23 /* Set this if the source data was used as a destination in a previous CP DMA
24 * packet. It's for preventing a read-after-write (RAW) hazard between two
25 * CP DMA packets. */
26 #define CP_DMA_RAW_WAIT (1 << 1)
27 #define CP_DMA_USE_L2 (1 << 2)
28 #define CP_DMA_CLEAR (1 << 3)
29
30 /* Alignment for optimal performance. */
31 #define SI_CPDMA_ALIGNMENT 32
32
33 /* The max number of bytes that can be copied per packet. */
34 static inline unsigned
cp_dma_max_byte_count(enum amd_gfx_level gfx_level)35 cp_dma_max_byte_count(enum amd_gfx_level gfx_level)
36 {
37 unsigned max = gfx_level >= GFX11 ? 32767
38 : gfx_level >= GFX9 ? S_415_BYTE_COUNT_GFX9(~0u)
39 : S_415_BYTE_COUNT_GFX6(~0u);
40
41 /* make it aligned for optimal performance */
42 return max & ~(SI_CPDMA_ALIGNMENT - 1);
43 }
44
45 /* Emit a CP DMA packet to do a copy from one buffer to another, or to clear
46 * a buffer. The size must fit in bits [20:0]. If CP_DMA_CLEAR is set, src_va is a 32-bit
47 * clear value.
48 */
49 static void
radv_cs_emit_cp_dma(struct radv_device * device,struct radeon_cmdbuf * cs,bool predicating,uint64_t dst_va,uint64_t src_va,unsigned size,unsigned flags)50 radv_cs_emit_cp_dma(struct radv_device *device, struct radeon_cmdbuf *cs, bool predicating, uint64_t dst_va,
51 uint64_t src_va, unsigned size, unsigned flags)
52 {
53 const struct radv_physical_device *pdev = radv_device_physical(device);
54 uint32_t header = 0, command = 0;
55
56 assert(size <= cp_dma_max_byte_count(pdev->info.gfx_level));
57
58 radeon_check_space(device->ws, cs, 9);
59 if (pdev->info.gfx_level >= GFX9)
60 command |= S_415_BYTE_COUNT_GFX9(size);
61 else
62 command |= S_415_BYTE_COUNT_GFX6(size);
63
64 /* Sync flags. */
65 if (flags & CP_DMA_SYNC)
66 header |= S_411_CP_SYNC(1);
67
68 if (flags & CP_DMA_RAW_WAIT)
69 command |= S_415_RAW_WAIT(1);
70
71 /* Src and dst flags. */
72 if (pdev->info.gfx_level >= GFX9 && !(flags & CP_DMA_CLEAR) && src_va == dst_va)
73 header |= S_411_DST_SEL(V_411_NOWHERE); /* prefetch only */
74 else if (flags & CP_DMA_USE_L2)
75 header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2);
76
77 if (flags & CP_DMA_CLEAR)
78 header |= S_411_SRC_SEL(V_411_DATA);
79 else if (flags & CP_DMA_USE_L2)
80 header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
81
82 if (pdev->info.gfx_level >= GFX7) {
83 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating));
84 radeon_emit(cs, header);
85 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
86 radeon_emit(cs, src_va >> 32); /* SRC_ADDR_HI [31:0] */
87 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
88 radeon_emit(cs, dst_va >> 32); /* DST_ADDR_HI [31:0] */
89 radeon_emit(cs, command);
90 } else {
91 assert(!(flags & CP_DMA_USE_L2));
92 header |= S_411_SRC_ADDR_HI(src_va >> 32);
93 radeon_emit(cs, PKT3(PKT3_CP_DMA, 4, predicating));
94 radeon_emit(cs, src_va); /* SRC_ADDR_LO [31:0] */
95 radeon_emit(cs, header); /* SRC_ADDR_HI [15:0] + flags. */
96 radeon_emit(cs, dst_va); /* DST_ADDR_LO [31:0] */
97 radeon_emit(cs, (dst_va >> 32) & 0xffff); /* DST_ADDR_HI [15:0] */
98 radeon_emit(cs, command);
99 }
100 }
101
102 static void
radv_emit_cp_dma(struct radv_cmd_buffer * cmd_buffer,uint64_t dst_va,uint64_t src_va,unsigned size,unsigned flags)103 radv_emit_cp_dma(struct radv_cmd_buffer *cmd_buffer, uint64_t dst_va, uint64_t src_va, unsigned size, unsigned flags)
104 {
105 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
106 struct radeon_cmdbuf *cs = cmd_buffer->cs;
107 bool predicating = cmd_buffer->state.predicating;
108
109 radv_cs_emit_cp_dma(device, cs, predicating, dst_va, src_va, size, flags);
110
111 /* CP DMA is executed in ME, but index buffers are read by PFP.
112 * This ensures that ME (CP DMA) is idle before PFP starts fetching
113 * indices. If we wanted to execute CP DMA in PFP, this packet
114 * should precede it.
115 */
116 if (flags & CP_DMA_SYNC) {
117 if (cmd_buffer->qf == RADV_QUEUE_GENERAL) {
118 radeon_emit(cs, PKT3(PKT3_PFP_SYNC_ME, 0, cmd_buffer->state.predicating));
119 radeon_emit(cs, 0);
120 }
121
122 /* CP will see the sync flag and wait for all DMAs to complete. */
123 cmd_buffer->state.dma_is_busy = false;
124 }
125
126 if (radv_device_fault_detection_enabled(device))
127 radv_cmd_buffer_trace_emit(cmd_buffer);
128 }
129
130 void
radv_cs_cp_dma_prefetch(const struct radv_device * device,struct radeon_cmdbuf * cs,uint64_t va,unsigned size,bool predicating)131 radv_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, unsigned size,
132 bool predicating)
133 {
134 const struct radv_physical_device *pdev = radv_device_physical(device);
135 struct radeon_winsys *ws = device->ws;
136 enum amd_gfx_level gfx_level = pdev->info.gfx_level;
137 uint32_t header = 0, command = 0;
138
139 if (gfx_level >= GFX11)
140 size = MIN2(size, 32768 - SI_CPDMA_ALIGNMENT);
141
142 assert(size <= cp_dma_max_byte_count(gfx_level));
143
144 radeon_check_space(ws, cs, 9);
145
146 uint64_t aligned_va = va & ~(SI_CPDMA_ALIGNMENT - 1);
147 uint64_t aligned_size = ((va + size + SI_CPDMA_ALIGNMENT - 1) & ~(SI_CPDMA_ALIGNMENT - 1)) - aligned_va;
148
149 if (gfx_level >= GFX9) {
150 command |= S_415_BYTE_COUNT_GFX9(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX9(1);
151 header |= S_411_DST_SEL(V_411_NOWHERE);
152 } else {
153 command |= S_415_BYTE_COUNT_GFX6(aligned_size) | S_415_DISABLE_WR_CONFIRM_GFX6(1);
154 header |= S_411_DST_SEL(V_411_DST_ADDR_TC_L2);
155 }
156
157 header |= S_411_SRC_SEL(V_411_SRC_ADDR_TC_L2);
158
159 radeon_emit(cs, PKT3(PKT3_DMA_DATA, 5, predicating));
160 radeon_emit(cs, header);
161 radeon_emit(cs, aligned_va); /* SRC_ADDR_LO [31:0] */
162 radeon_emit(cs, aligned_va >> 32); /* SRC_ADDR_HI [31:0] */
163 radeon_emit(cs, aligned_va); /* DST_ADDR_LO [31:0] */
164 radeon_emit(cs, aligned_va >> 32); /* DST_ADDR_HI [31:0] */
165 radeon_emit(cs, command);
166 }
167
168 void
radv_cp_dma_prefetch(struct radv_cmd_buffer * cmd_buffer,uint64_t va,unsigned size)169 radv_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size)
170 {
171 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
172
173 radv_cs_cp_dma_prefetch(device, cmd_buffer->cs, va, size, cmd_buffer->state.predicating);
174
175 if (radv_device_fault_detection_enabled(device))
176 radv_cmd_buffer_trace_emit(cmd_buffer);
177 }
178
179 static void
radv_cp_dma_prepare(struct radv_cmd_buffer * cmd_buffer,uint64_t byte_count,uint64_t remaining_size,unsigned * flags)180 radv_cp_dma_prepare(struct radv_cmd_buffer *cmd_buffer, uint64_t byte_count, uint64_t remaining_size, unsigned *flags)
181 {
182
183 /* Flush the caches for the first copy only.
184 * Also wait for the previous CP DMA operations.
185 */
186 if (cmd_buffer->state.flush_bits) {
187 radv_emit_cache_flush(cmd_buffer);
188 *flags |= CP_DMA_RAW_WAIT;
189 }
190
191 /* Do the synchronization after the last dma, so that all data
192 * is written to memory.
193 */
194 if (byte_count == remaining_size)
195 *flags |= CP_DMA_SYNC;
196 }
197
198 static void
radv_cp_dma_realign_engine(struct radv_cmd_buffer * cmd_buffer,unsigned size)199 radv_cp_dma_realign_engine(struct radv_cmd_buffer *cmd_buffer, unsigned size)
200 {
201 uint64_t va;
202 uint32_t offset;
203 unsigned dma_flags = 0;
204 unsigned buf_size = SI_CPDMA_ALIGNMENT * 2;
205 void *ptr;
206
207 assert(size < SI_CPDMA_ALIGNMENT);
208
209 radv_cmd_buffer_upload_alloc(cmd_buffer, buf_size, &offset, &ptr);
210
211 va = radv_buffer_get_va(cmd_buffer->upload.upload_bo);
212 va += offset;
213
214 radv_cp_dma_prepare(cmd_buffer, size, size, &dma_flags);
215
216 radv_emit_cp_dma(cmd_buffer, va, va + SI_CPDMA_ALIGNMENT, size, dma_flags);
217 }
218
219 void
radv_cp_dma_buffer_copy(struct radv_cmd_buffer * cmd_buffer,uint64_t src_va,uint64_t dest_va,uint64_t size)220 radv_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size)
221 {
222 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
223 const struct radv_physical_device *pdev = radv_device_physical(device);
224 enum amd_gfx_level gfx_level = pdev->info.gfx_level;
225 uint64_t main_src_va, main_dest_va;
226 uint64_t skipped_size = 0, realign_size = 0;
227
228 /* Assume that we are not going to sync after the last DMA operation. */
229 cmd_buffer->state.dma_is_busy = true;
230
231 if (pdev->info.family <= CHIP_CARRIZO || pdev->info.family == CHIP_STONEY) {
232 /* If the size is not aligned, we must add a dummy copy at the end
233 * just to align the internal counter. Otherwise, the DMA engine
234 * would slow down by an order of magnitude for following copies.
235 */
236 if (size % SI_CPDMA_ALIGNMENT)
237 realign_size = SI_CPDMA_ALIGNMENT - (size % SI_CPDMA_ALIGNMENT);
238
239 /* If the copy begins unaligned, we must start copying from the next
240 * aligned block and the skipped part should be copied after everything
241 * else has been copied. Only the src alignment matters, not dst.
242 */
243 if (src_va % SI_CPDMA_ALIGNMENT) {
244 skipped_size = SI_CPDMA_ALIGNMENT - (src_va % SI_CPDMA_ALIGNMENT);
245 /* The main part will be skipped if the size is too small. */
246 skipped_size = MIN2(skipped_size, size);
247 size -= skipped_size;
248 }
249 }
250 main_src_va = src_va + skipped_size;
251 main_dest_va = dest_va + skipped_size;
252
253 while (size) {
254 unsigned dma_flags = 0;
255 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level));
256
257 if (pdev->info.gfx_level >= GFX9) {
258 /* DMA operations via L2 are coherent and faster.
259 * TODO: GFX7-GFX8 should also support this but it
260 * requires tests/benchmarks.
261 *
262 * Also enable on GFX9 so we can use L2 at rest on GFX9+. On Raven
263 * this didn't seem to be worse.
264 *
265 * Note that we only use CP DMA for sizes < RADV_BUFFER_OPS_CS_THRESHOLD,
266 * which is 4k at the moment, so this is really unlikely to cause
267 * significant thrashing.
268 */
269 dma_flags |= CP_DMA_USE_L2;
270 }
271
272 radv_cp_dma_prepare(cmd_buffer, byte_count, size + skipped_size + realign_size, &dma_flags);
273
274 dma_flags &= ~CP_DMA_SYNC;
275
276 radv_emit_cp_dma(cmd_buffer, main_dest_va, main_src_va, byte_count, dma_flags);
277
278 size -= byte_count;
279 main_src_va += byte_count;
280 main_dest_va += byte_count;
281 }
282
283 if (skipped_size) {
284 unsigned dma_flags = 0;
285
286 radv_cp_dma_prepare(cmd_buffer, skipped_size, size + skipped_size + realign_size, &dma_flags);
287
288 radv_emit_cp_dma(cmd_buffer, dest_va, src_va, skipped_size, dma_flags);
289 }
290 if (realign_size)
291 radv_cp_dma_realign_engine(cmd_buffer, realign_size);
292 }
293
294 void
radv_cp_dma_clear_buffer(struct radv_cmd_buffer * cmd_buffer,uint64_t va,uint64_t size,unsigned value)295 radv_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value)
296 {
297 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
298 const struct radv_physical_device *pdev = radv_device_physical(device);
299
300 if (!size)
301 return;
302
303 assert(va % 4 == 0 && size % 4 == 0);
304
305 enum amd_gfx_level gfx_level = pdev->info.gfx_level;
306
307 /* Assume that we are not going to sync after the last DMA operation. */
308 cmd_buffer->state.dma_is_busy = true;
309
310 while (size) {
311 unsigned byte_count = MIN2(size, cp_dma_max_byte_count(gfx_level));
312 unsigned dma_flags = CP_DMA_CLEAR;
313
314 if (pdev->info.gfx_level >= GFX9) {
315 /* DMA operations via L2 are coherent and faster.
316 * TODO: GFX7-GFX8 should also support this but it
317 * requires tests/benchmarks.
318 *
319 * Also enable on GFX9 so we can use L2 at rest on GFX9+.
320 */
321 dma_flags |= CP_DMA_USE_L2;
322 }
323
324 radv_cp_dma_prepare(cmd_buffer, byte_count, size, &dma_flags);
325
326 /* Emit the clear packet. */
327 radv_emit_cp_dma(cmd_buffer, va, value, byte_count, dma_flags);
328
329 size -= byte_count;
330 va += byte_count;
331 }
332 }
333
334 void
radv_cp_dma_wait_for_idle(struct radv_cmd_buffer * cmd_buffer)335 radv_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer)
336 {
337 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
338 const struct radv_physical_device *pdev = radv_device_physical(device);
339
340 if (pdev->info.gfx_level < GFX7)
341 return;
342
343 if (!cmd_buffer->state.dma_is_busy)
344 return;
345
346 /* Issue a dummy DMA that copies zero bytes.
347 *
348 * The DMA engine will see that there's no work to do and skip this
349 * DMA request, however, the CP will see the sync flag and still wait
350 * for all DMAs to complete.
351 */
352 radv_emit_cp_dma(cmd_buffer, 0, 0, 0, CP_DMA_SYNC);
353
354 cmd_buffer->state.dma_is_busy = false;
355 }
356