1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * AMD Passthrough DMA device driver
4 * -- Based on the CCP driver
5 *
6 * Copyright (C) 2016,2021 Advanced Micro Devices, Inc.
7 *
8 * Author: Sanjay R Mehta <[email protected]>
9 * Author: Gary R Hook <[email protected]>
10 */
11
12 #include <linux/bitfield.h>
13 #include "ptdma.h"
14 #include "../ae4dma/ae4dma.h"
15 #include "../../dmaengine.h"
16
17 static char *ae4_error_codes[] = {
18 "",
19 "ERR 01: INVALID HEADER DW0",
20 "ERR 02: INVALID STATUS",
21 "ERR 03: INVALID LENGTH - 4 BYTE ALIGNMENT",
22 "ERR 04: INVALID SRC ADDR - 4 BYTE ALIGNMENT",
23 "ERR 05: INVALID DST ADDR - 4 BYTE ALIGNMENT",
24 "ERR 06: INVALID ALIGNMENT",
25 "ERR 07: INVALID DESCRIPTOR",
26 };
27
ae4_log_error(struct pt_device * d,int e)28 static void ae4_log_error(struct pt_device *d, int e)
29 {
30 /* ERR 01 - 07 represents Invalid AE4 errors */
31 if (e <= 7)
32 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", ae4_error_codes[e], e);
33 /* ERR 08 - 15 represents Invalid Descriptor errors */
34 else if (e > 7 && e <= 15)
35 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "INVALID DESCRIPTOR", e);
36 /* ERR 16 - 31 represents Firmware errors */
37 else if (e > 15 && e <= 31)
38 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FIRMWARE ERROR", e);
39 /* ERR 32 - 63 represents Fatal errors */
40 else if (e > 31 && e <= 63)
41 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FATAL ERROR", e);
42 /* ERR 64 - 255 represents PTE errors */
43 else if (e > 63 && e <= 255)
44 dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "PTE ERROR", e);
45 else
46 dev_info(d->dev, "Unknown AE4DMA error");
47 }
48
ae4_check_status_error(struct ae4_cmd_queue * ae4cmd_q,int idx)49 void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx)
50 {
51 struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q;
52 struct ae4dma_desc desc;
53 u8 status;
54
55 memcpy(&desc, &cmd_q->qbase[idx], sizeof(struct ae4dma_desc));
56 status = desc.dw1.status;
57 if (status && status != AE4_DESC_COMPLETED) {
58 cmd_q->cmd_error = desc.dw1.err_code;
59 if (cmd_q->cmd_error)
60 ae4_log_error(cmd_q->pt, cmd_q->cmd_error);
61 }
62 }
63 EXPORT_SYMBOL_GPL(ae4_check_status_error);
64
to_pt_chan(struct dma_chan * dma_chan)65 static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan)
66 {
67 return container_of(dma_chan, struct pt_dma_chan, vc.chan);
68 }
69
to_pt_desc(struct virt_dma_desc * vd)70 static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd)
71 {
72 return container_of(vd, struct pt_dma_desc, vd);
73 }
74
pt_free_chan_resources(struct dma_chan * dma_chan)75 static void pt_free_chan_resources(struct dma_chan *dma_chan)
76 {
77 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
78
79 vchan_free_chan_resources(&chan->vc);
80 }
81
pt_synchronize(struct dma_chan * dma_chan)82 static void pt_synchronize(struct dma_chan *dma_chan)
83 {
84 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
85
86 vchan_synchronize(&chan->vc);
87 }
88
pt_do_cleanup(struct virt_dma_desc * vd)89 static void pt_do_cleanup(struct virt_dma_desc *vd)
90 {
91 struct pt_dma_desc *desc = to_pt_desc(vd);
92 struct pt_device *pt = desc->pt;
93
94 kmem_cache_free(pt->dma_desc_cache, desc);
95 }
96
pt_get_cmd_queue(struct pt_device * pt,struct pt_dma_chan * chan)97 static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma_chan *chan)
98 {
99 struct ae4_cmd_queue *ae4cmd_q;
100 struct pt_cmd_queue *cmd_q;
101 struct ae4_device *ae4;
102
103 if (pt->ver == AE4_DMA_VERSION) {
104 ae4 = container_of(pt, struct ae4_device, pt);
105 ae4cmd_q = &ae4->ae4cmd_q[chan->id];
106 cmd_q = &ae4cmd_q->cmd_q;
107 } else {
108 cmd_q = &pt->cmd_q;
109 }
110
111 return cmd_q;
112 }
113
ae4_core_execute_cmd(struct ae4dma_desc * desc,struct ae4_cmd_queue * ae4cmd_q)114 static int ae4_core_execute_cmd(struct ae4dma_desc *desc, struct ae4_cmd_queue *ae4cmd_q)
115 {
116 bool soc = FIELD_GET(DWORD0_SOC, desc->dwouv.dw0);
117 struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q;
118
119 if (soc) {
120 desc->dwouv.dw0 |= FIELD_PREP(DWORD0_IOC, desc->dwouv.dw0);
121 desc->dwouv.dw0 &= ~DWORD0_SOC;
122 }
123
124 mutex_lock(&ae4cmd_q->cmd_lock);
125 memcpy(&cmd_q->qbase[ae4cmd_q->tail_wi], desc, sizeof(struct ae4dma_desc));
126 ae4cmd_q->q_cmd_count++;
127 ae4cmd_q->tail_wi = (ae4cmd_q->tail_wi + 1) % CMD_Q_LEN;
128 writel(ae4cmd_q->tail_wi, cmd_q->reg_control + AE4_WR_IDX_OFF);
129 mutex_unlock(&ae4cmd_q->cmd_lock);
130
131 wake_up(&ae4cmd_q->q_w);
132
133 return 0;
134 }
135
pt_core_perform_passthru_ae4(struct pt_cmd_queue * cmd_q,struct pt_passthru_engine * pt_engine)136 static int pt_core_perform_passthru_ae4(struct pt_cmd_queue *cmd_q,
137 struct pt_passthru_engine *pt_engine)
138 {
139 struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q);
140 struct ae4dma_desc desc;
141
142 cmd_q->cmd_error = 0;
143 cmd_q->total_pt_ops++;
144 memset(&desc, 0, sizeof(desc));
145 desc.dwouv.dws.byte0 = CMD_AE4_DESC_DW0_VAL;
146
147 desc.dw1.status = 0;
148 desc.dw1.err_code = 0;
149 desc.dw1.desc_id = 0;
150
151 desc.length = pt_engine->src_len;
152
153 desc.src_lo = upper_32_bits(pt_engine->src_dma);
154 desc.src_hi = lower_32_bits(pt_engine->src_dma);
155 desc.dst_lo = upper_32_bits(pt_engine->dst_dma);
156 desc.dst_hi = lower_32_bits(pt_engine->dst_dma);
157
158 return ae4_core_execute_cmd(&desc, ae4cmd_q);
159 }
160
pt_dma_start_desc(struct pt_dma_desc * desc,struct pt_dma_chan * chan)161 static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan)
162 {
163 struct pt_passthru_engine *pt_engine;
164 struct pt_device *pt;
165 struct pt_cmd *pt_cmd;
166 struct pt_cmd_queue *cmd_q;
167
168 desc->issued_to_hw = 1;
169
170 pt_cmd = &desc->pt_cmd;
171 pt = pt_cmd->pt;
172
173 cmd_q = pt_get_cmd_queue(pt, chan);
174
175 pt_engine = &pt_cmd->passthru;
176
177 pt->tdata.cmd = pt_cmd;
178
179 /* Execute the command */
180 if (pt->ver == AE4_DMA_VERSION)
181 pt_cmd->ret = pt_core_perform_passthru_ae4(cmd_q, pt_engine);
182 else
183 pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine);
184
185 return 0;
186 }
187
pt_next_dma_desc(struct pt_dma_chan * chan)188 static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan)
189 {
190 /* Get the next DMA descriptor on the active list */
191 struct virt_dma_desc *vd = vchan_next_desc(&chan->vc);
192
193 return vd ? to_pt_desc(vd) : NULL;
194 }
195
pt_handle_active_desc(struct pt_dma_chan * chan,struct pt_dma_desc * desc)196 static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan,
197 struct pt_dma_desc *desc)
198 {
199 struct dma_async_tx_descriptor *tx_desc;
200 struct virt_dma_desc *vd;
201 struct pt_device *pt;
202 unsigned long flags;
203
204 pt = chan->pt;
205 /* Loop over descriptors until one is found with commands */
206 do {
207 if (desc) {
208 if (!desc->issued_to_hw) {
209 /* No errors, keep going */
210 if (desc->status != DMA_ERROR)
211 return desc;
212 }
213
214 tx_desc = &desc->vd.tx;
215 vd = &desc->vd;
216 } else {
217 tx_desc = NULL;
218 }
219
220 spin_lock_irqsave(&chan->vc.lock, flags);
221
222 if (pt->ver != AE4_DMA_VERSION && desc) {
223 if (desc->status != DMA_COMPLETE) {
224 if (desc->status != DMA_ERROR)
225 desc->status = DMA_COMPLETE;
226
227 dma_cookie_complete(tx_desc);
228 dma_descriptor_unmap(tx_desc);
229 list_del(&desc->vd.node);
230 } else {
231 /* Don't handle it twice */
232 tx_desc = NULL;
233 }
234 }
235
236 desc = pt_next_dma_desc(chan);
237
238 spin_unlock_irqrestore(&chan->vc.lock, flags);
239
240 if (pt->ver != AE4_DMA_VERSION && tx_desc) {
241 dmaengine_desc_get_callback_invoke(tx_desc, NULL);
242 dma_run_dependencies(tx_desc);
243 vchan_vdesc_fini(vd);
244 }
245 } while (desc);
246
247 return NULL;
248 }
249
ae4_core_queue_full(struct pt_cmd_queue * cmd_q)250 static inline bool ae4_core_queue_full(struct pt_cmd_queue *cmd_q)
251 {
252 u32 front_wi = readl(cmd_q->reg_control + AE4_WR_IDX_OFF);
253 u32 rear_ri = readl(cmd_q->reg_control + AE4_RD_IDX_OFF);
254
255 if (((MAX_CMD_QLEN + front_wi - rear_ri) % MAX_CMD_QLEN) >= (MAX_CMD_QLEN - 1))
256 return true;
257
258 return false;
259 }
260
pt_cmd_callback(void * data,int err)261 static void pt_cmd_callback(void *data, int err)
262 {
263 struct pt_dma_desc *desc = data;
264 struct ae4_cmd_queue *ae4cmd_q;
265 struct dma_chan *dma_chan;
266 struct pt_dma_chan *chan;
267 struct ae4_device *ae4;
268 struct pt_device *pt;
269 int ret;
270
271 if (err == -EINPROGRESS)
272 return;
273
274 dma_chan = desc->vd.tx.chan;
275 chan = to_pt_chan(dma_chan);
276 pt = chan->pt;
277
278 if (err)
279 desc->status = DMA_ERROR;
280
281 while (true) {
282 if (pt->ver == AE4_DMA_VERSION) {
283 ae4 = container_of(pt, struct ae4_device, pt);
284 ae4cmd_q = &ae4->ae4cmd_q[chan->id];
285
286 if (ae4cmd_q->q_cmd_count >= (CMD_Q_LEN - 1) ||
287 ae4_core_queue_full(&ae4cmd_q->cmd_q)) {
288 wake_up(&ae4cmd_q->q_w);
289
290 if (wait_for_completion_timeout(&ae4cmd_q->cmp,
291 msecs_to_jiffies(AE4_TIME_OUT))
292 == 0) {
293 dev_err(pt->dev, "TIMEOUT %d:\n", ae4cmd_q->id);
294 break;
295 }
296
297 reinit_completion(&ae4cmd_q->cmp);
298 continue;
299 }
300 }
301
302 /* Check for DMA descriptor completion */
303 desc = pt_handle_active_desc(chan, desc);
304
305 /* Don't submit cmd if no descriptor or DMA is paused */
306 if (!desc)
307 break;
308
309 ret = pt_dma_start_desc(desc, chan);
310 if (!ret)
311 break;
312
313 desc->status = DMA_ERROR;
314 }
315 }
316
pt_alloc_dma_desc(struct pt_dma_chan * chan,unsigned long flags)317 static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan,
318 unsigned long flags)
319 {
320 struct pt_dma_desc *desc;
321
322 desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT);
323 if (!desc)
324 return NULL;
325
326 vchan_tx_prep(&chan->vc, &desc->vd, flags);
327
328 desc->pt = chan->pt;
329 desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT);
330 desc->issued_to_hw = 0;
331 desc->status = DMA_IN_PROGRESS;
332
333 return desc;
334 }
335
pt_cmd_callback_work(void * data,int err)336 static void pt_cmd_callback_work(void *data, int err)
337 {
338 struct dma_async_tx_descriptor *tx_desc;
339 struct pt_dma_desc *desc = data;
340 struct dma_chan *dma_chan;
341 struct virt_dma_desc *vd;
342 struct pt_dma_chan *chan;
343 unsigned long flags;
344
345 dma_chan = desc->vd.tx.chan;
346 chan = to_pt_chan(dma_chan);
347
348 if (err == -EINPROGRESS)
349 return;
350
351 tx_desc = &desc->vd.tx;
352 vd = &desc->vd;
353
354 if (err)
355 desc->status = DMA_ERROR;
356
357 spin_lock_irqsave(&chan->vc.lock, flags);
358 if (desc) {
359 if (desc->status != DMA_COMPLETE) {
360 if (desc->status != DMA_ERROR)
361 desc->status = DMA_COMPLETE;
362
363 dma_cookie_complete(tx_desc);
364 dma_descriptor_unmap(tx_desc);
365 } else {
366 tx_desc = NULL;
367 }
368 }
369 spin_unlock_irqrestore(&chan->vc.lock, flags);
370
371 if (tx_desc) {
372 dmaengine_desc_get_callback_invoke(tx_desc, NULL);
373 dma_run_dependencies(tx_desc);
374 list_del(&desc->vd.node);
375 vchan_vdesc_fini(vd);
376 }
377 }
378
pt_create_desc(struct dma_chan * dma_chan,dma_addr_t dst,dma_addr_t src,unsigned int len,unsigned long flags)379 static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan,
380 dma_addr_t dst,
381 dma_addr_t src,
382 unsigned int len,
383 unsigned long flags)
384 {
385 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
386 struct pt_passthru_engine *pt_engine;
387 struct pt_device *pt = chan->pt;
388 struct ae4_cmd_queue *ae4cmd_q;
389 struct pt_dma_desc *desc;
390 struct ae4_device *ae4;
391 struct pt_cmd *pt_cmd;
392
393 desc = pt_alloc_dma_desc(chan, flags);
394 if (!desc)
395 return NULL;
396
397 pt_cmd = &desc->pt_cmd;
398 pt_cmd->pt = pt;
399 pt_engine = &pt_cmd->passthru;
400 pt_cmd->engine = PT_ENGINE_PASSTHRU;
401 pt_engine->src_dma = src;
402 pt_engine->dst_dma = dst;
403 pt_engine->src_len = len;
404 pt_cmd->pt_cmd_callback = pt_cmd_callback;
405 pt_cmd->data = desc;
406
407 desc->len = len;
408
409 if (pt->ver == AE4_DMA_VERSION) {
410 pt_cmd->pt_cmd_callback = pt_cmd_callback_work;
411 ae4 = container_of(pt, struct ae4_device, pt);
412 ae4cmd_q = &ae4->ae4cmd_q[chan->id];
413 mutex_lock(&ae4cmd_q->cmd_lock);
414 list_add_tail(&pt_cmd->entry, &ae4cmd_q->cmd);
415 mutex_unlock(&ae4cmd_q->cmd_lock);
416 }
417
418 return desc;
419 }
420
421 static struct dma_async_tx_descriptor *
pt_prep_dma_memcpy(struct dma_chan * dma_chan,dma_addr_t dst,dma_addr_t src,size_t len,unsigned long flags)422 pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst,
423 dma_addr_t src, size_t len, unsigned long flags)
424 {
425 struct pt_dma_desc *desc;
426
427 desc = pt_create_desc(dma_chan, dst, src, len, flags);
428 if (!desc)
429 return NULL;
430
431 return &desc->vd.tx;
432 }
433
434 static struct dma_async_tx_descriptor *
pt_prep_dma_interrupt(struct dma_chan * dma_chan,unsigned long flags)435 pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags)
436 {
437 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
438 struct pt_dma_desc *desc;
439
440 desc = pt_alloc_dma_desc(chan, flags);
441 if (!desc)
442 return NULL;
443
444 return &desc->vd.tx;
445 }
446
pt_issue_pending(struct dma_chan * dma_chan)447 static void pt_issue_pending(struct dma_chan *dma_chan)
448 {
449 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
450 struct pt_dma_desc *desc;
451 struct pt_device *pt;
452 unsigned long flags;
453 bool engine_is_idle = true;
454
455 pt = chan->pt;
456
457 spin_lock_irqsave(&chan->vc.lock, flags);
458
459 desc = pt_next_dma_desc(chan);
460 if (desc && pt->ver != AE4_DMA_VERSION)
461 engine_is_idle = false;
462
463 vchan_issue_pending(&chan->vc);
464
465 desc = pt_next_dma_desc(chan);
466
467 spin_unlock_irqrestore(&chan->vc.lock, flags);
468
469 /* If there was nothing active, start processing */
470 if (engine_is_idle && desc)
471 pt_cmd_callback(desc, 0);
472 }
473
pt_check_status_trans_ae4(struct pt_device * pt,struct pt_cmd_queue * cmd_q)474 static void pt_check_status_trans_ae4(struct pt_device *pt, struct pt_cmd_queue *cmd_q)
475 {
476 struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q);
477 int i;
478
479 for (i = 0; i < CMD_Q_LEN; i++)
480 ae4_check_status_error(ae4cmd_q, i);
481 }
482
483 static enum dma_status
pt_tx_status(struct dma_chan * c,dma_cookie_t cookie,struct dma_tx_state * txstate)484 pt_tx_status(struct dma_chan *c, dma_cookie_t cookie,
485 struct dma_tx_state *txstate)
486 {
487 struct pt_dma_chan *chan = to_pt_chan(c);
488 struct pt_device *pt = chan->pt;
489 struct pt_cmd_queue *cmd_q;
490
491 cmd_q = pt_get_cmd_queue(pt, chan);
492
493 if (pt->ver == AE4_DMA_VERSION)
494 pt_check_status_trans_ae4(pt, cmd_q);
495 else
496 pt_check_status_trans(pt, cmd_q);
497
498 return dma_cookie_status(c, cookie, txstate);
499 }
500
pt_pause(struct dma_chan * dma_chan)501 static int pt_pause(struct dma_chan *dma_chan)
502 {
503 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
504 struct pt_device *pt = chan->pt;
505 struct pt_cmd_queue *cmd_q;
506 unsigned long flags;
507
508 spin_lock_irqsave(&chan->vc.lock, flags);
509 cmd_q = pt_get_cmd_queue(pt, chan);
510 pt_stop_queue(cmd_q);
511 spin_unlock_irqrestore(&chan->vc.lock, flags);
512
513 return 0;
514 }
515
pt_resume(struct dma_chan * dma_chan)516 static int pt_resume(struct dma_chan *dma_chan)
517 {
518 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
519 struct pt_dma_desc *desc = NULL;
520 struct pt_device *pt = chan->pt;
521 struct pt_cmd_queue *cmd_q;
522 unsigned long flags;
523
524 spin_lock_irqsave(&chan->vc.lock, flags);
525 cmd_q = pt_get_cmd_queue(pt, chan);
526 pt_start_queue(cmd_q);
527 desc = pt_next_dma_desc(chan);
528 spin_unlock_irqrestore(&chan->vc.lock, flags);
529
530 /* If there was something active, re-start */
531 if (desc)
532 pt_cmd_callback(desc, 0);
533
534 return 0;
535 }
536
pt_terminate_all(struct dma_chan * dma_chan)537 static int pt_terminate_all(struct dma_chan *dma_chan)
538 {
539 struct pt_dma_chan *chan = to_pt_chan(dma_chan);
540 struct pt_device *pt = chan->pt;
541 struct pt_cmd_queue *cmd_q;
542 unsigned long flags;
543 LIST_HEAD(head);
544
545 cmd_q = pt_get_cmd_queue(pt, chan);
546 if (pt->ver == AE4_DMA_VERSION)
547 pt_stop_queue(cmd_q);
548 else
549 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010);
550
551 spin_lock_irqsave(&chan->vc.lock, flags);
552 vchan_get_all_descriptors(&chan->vc, &head);
553 spin_unlock_irqrestore(&chan->vc.lock, flags);
554
555 vchan_dma_desc_free_list(&chan->vc, &head);
556 vchan_free_chan_resources(&chan->vc);
557
558 return 0;
559 }
560
pt_dmaengine_register(struct pt_device * pt)561 int pt_dmaengine_register(struct pt_device *pt)
562 {
563 struct dma_device *dma_dev = &pt->dma_dev;
564 struct ae4_cmd_queue *ae4cmd_q = NULL;
565 struct ae4_device *ae4 = NULL;
566 struct pt_dma_chan *chan;
567 char *desc_cache_name;
568 char *cmd_cache_name;
569 int ret, i;
570
571 if (pt->ver == AE4_DMA_VERSION)
572 ae4 = container_of(pt, struct ae4_device, pt);
573
574 if (ae4)
575 pt->pt_dma_chan = devm_kcalloc(pt->dev, ae4->cmd_q_count,
576 sizeof(*pt->pt_dma_chan), GFP_KERNEL);
577 else
578 pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan),
579 GFP_KERNEL);
580
581 if (!pt->pt_dma_chan)
582 return -ENOMEM;
583
584 cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL,
585 "%s-dmaengine-cmd-cache",
586 dev_name(pt->dev));
587 if (!cmd_cache_name)
588 return -ENOMEM;
589
590 desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL,
591 "%s-dmaengine-desc-cache",
592 dev_name(pt->dev));
593 if (!desc_cache_name) {
594 ret = -ENOMEM;
595 goto err_cache;
596 }
597
598 pt->dma_desc_cache = kmem_cache_create(desc_cache_name,
599 sizeof(struct pt_dma_desc), 0,
600 SLAB_HWCACHE_ALIGN, NULL);
601 if (!pt->dma_desc_cache) {
602 ret = -ENOMEM;
603 goto err_cache;
604 }
605
606 dma_dev->dev = pt->dev;
607 dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
608 dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES;
609 dma_dev->directions = DMA_MEM_TO_MEM;
610 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
611 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
612 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
613
614 /*
615 * PTDMA is intended to be used with the AMD NTB devices, hence
616 * marking it as DMA_PRIVATE.
617 */
618 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
619
620 INIT_LIST_HEAD(&dma_dev->channels);
621
622 /* Set base and prep routines */
623 dma_dev->device_free_chan_resources = pt_free_chan_resources;
624 dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy;
625 dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt;
626 dma_dev->device_issue_pending = pt_issue_pending;
627 dma_dev->device_tx_status = pt_tx_status;
628 dma_dev->device_pause = pt_pause;
629 dma_dev->device_resume = pt_resume;
630 dma_dev->device_terminate_all = pt_terminate_all;
631 dma_dev->device_synchronize = pt_synchronize;
632
633 if (ae4) {
634 for (i = 0; i < ae4->cmd_q_count; i++) {
635 chan = pt->pt_dma_chan + i;
636 ae4cmd_q = &ae4->ae4cmd_q[i];
637 chan->id = ae4cmd_q->id;
638 chan->pt = pt;
639 chan->vc.desc_free = pt_do_cleanup;
640 vchan_init(&chan->vc, dma_dev);
641 }
642 } else {
643 chan = pt->pt_dma_chan;
644 chan->pt = pt;
645 chan->vc.desc_free = pt_do_cleanup;
646 vchan_init(&chan->vc, dma_dev);
647 }
648
649 ret = dma_async_device_register(dma_dev);
650 if (ret)
651 goto err_reg;
652
653 return 0;
654
655 err_reg:
656 kmem_cache_destroy(pt->dma_desc_cache);
657
658 err_cache:
659 kmem_cache_destroy(pt->dma_cmd_cache);
660
661 return ret;
662 }
663 EXPORT_SYMBOL_GPL(pt_dmaengine_register);
664
pt_dmaengine_unregister(struct pt_device * pt)665 void pt_dmaengine_unregister(struct pt_device *pt)
666 {
667 struct dma_device *dma_dev = &pt->dma_dev;
668
669 dma_async_device_unregister(dma_dev);
670
671 kmem_cache_destroy(pt->dma_desc_cache);
672 kmem_cache_destroy(pt->dma_cmd_cache);
673 }
674