3  * IOMMU API for RISC-V IOMMU implementations.
13 #define pr_fmt(fmt) "riscv-iommu: " fmt
18 #include <linux/iommu.h>
23 #include "../iommu-pages.h"
24 #include "iommu-bits.h"
25 #include "iommu.h"
37 /* RISC-V IOMMU PPN <> PHYS address conversions, PHYS <=> PPN[53:10] */
42 	iommu_get_iommu_dev(dev, struct riscv_iommu_device, iommu)
44 /* IOMMU PSCID allocation namespace. */
69 static void *riscv_iommu_get_pages(struct riscv_iommu_device *iommu, int order)  in riscv_iommu_get_pages()  argument
74 	addr = iommu_alloc_pages_node(dev_to_node(iommu->dev),  in riscv_iommu_get_pages()
90 	devres_add(iommu->dev, devres);  in riscv_iommu_get_pages()
95 static void riscv_iommu_free_pages(struct riscv_iommu_device *iommu, void *addr)  in riscv_iommu_free_pages()  argument
99 	devres_release(iommu->dev, riscv_iommu_devres_pages_release,  in riscv_iommu_free_pages()
130 static int riscv_iommu_queue_alloc(struct riscv_iommu_device *iommu,  in riscv_iommu_queue_alloc()  argument
141 	riscv_iommu_writeq(iommu, queue->qbr, RISCV_IOMMU_QUEUE_LOG2SZ_FIELD);  in riscv_iommu_queue_alloc()
142 	qb = riscv_iommu_readq(iommu, queue->qbr);  in riscv_iommu_queue_alloc()
162 		queue->base = devm_ioremap(iommu->dev, queue->phys, queue_size);  in riscv_iommu_queue_alloc()
168 			queue->base = riscv_iommu_get_pages(iommu, order);  in riscv_iommu_queue_alloc()
180 	riscv_iommu_writeq(iommu, queue->qbr, qb);  in riscv_iommu_queue_alloc()
181 	rb = riscv_iommu_readq(iommu, queue->qbr);  in riscv_iommu_queue_alloc()
183 		dev_err(iommu->dev, "queue #%u allocation failed\n", queue->qid);  in riscv_iommu_queue_alloc()
190 	dev_dbg(iommu->dev, "queue #%u allocated 2^%u entries",  in riscv_iommu_queue_alloc()
201 	if (riscv_iommu_readl(queue->iommu, RISCV_IOMMU_REG_IPSR) & Q_IPSR(queue))  in riscv_iommu_queue_ipsr()
207 static int riscv_iommu_queue_vec(struct riscv_iommu_device *iommu, int n)  in riscv_iommu_queue_vec()  argument
210 	return (iommu->icvec >> (n * 4)) & RISCV_IOMMU_ICVEC_CIV;  in riscv_iommu_queue_vec()
219 static int riscv_iommu_queue_enable(struct riscv_iommu_device *iommu,  in riscv_iommu_queue_enable()  argument
223 	const unsigned int irq = iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)];  in riscv_iommu_queue_enable()
227 	if (queue->iommu)  in riscv_iommu_queue_enable()
234 	queue->iommu = iommu;  in riscv_iommu_queue_enable()
237 				  dev_name(iommu->dev), queue);  in riscv_iommu_queue_enable()
239 		queue->iommu = NULL;  in riscv_iommu_queue_enable()
245 		riscv_iommu_writel(queue->iommu, Q_TAIL(queue), 0);  in riscv_iommu_queue_enable()
247 		riscv_iommu_writel(queue->iommu, Q_HEAD(queue), 0);  in riscv_iommu_queue_enable()
255 	riscv_iommu_writel(iommu, queue->qcr,  in riscv_iommu_queue_enable()
260 	riscv_iommu_readl_timeout(iommu, queue->qcr,  in riscv_iommu_queue_enable()
268 		riscv_iommu_writel(iommu, queue->qcr, 0);  in riscv_iommu_queue_enable()
270 		queue->iommu = NULL;  in riscv_iommu_queue_enable()
271 		dev_err(iommu->dev, "queue #%u failed to start\n", queue->qid);  in riscv_iommu_queue_enable()
276 	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));  in riscv_iommu_queue_enable()
287 	struct riscv_iommu_device *iommu = queue->iommu;  in riscv_iommu_queue_disable()  local
290 	if (!iommu)  in riscv_iommu_queue_disable()
293 	free_irq(iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)], queue);  in riscv_iommu_queue_disable()
294 	riscv_iommu_writel(iommu, queue->qcr, 0);  in riscv_iommu_queue_disable()
295 	riscv_iommu_readl_timeout(iommu, queue->qcr,  in riscv_iommu_queue_disable()
300 		dev_err(iommu->dev, "fail to disable hardware queue #%u, csr 0x%x\n",  in riscv_iommu_queue_disable()
303 	queue->iommu = NULL;  in riscv_iommu_queue_disable()
324 	if (riscv_iommu_readl_timeout(queue->iommu, Q_TAIL(queue),  in riscv_iommu_queue_consume()
327 		dev_err_once(queue->iommu->dev,  in riscv_iommu_queue_consume()
346 	riscv_iommu_writel(queue->iommu, Q_HEAD(queue), Q_ITEM(queue, head));  in riscv_iommu_queue_release()
356 	if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,  in riscv_iommu_queue_cons()
382  * Error handling for IOMMU hardware not responding in reasonable time
410 		if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,  in riscv_iommu_queue_send()
431 	riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1));  in riscv_iommu_queue_send()
447 	dev_err_once(queue->iommu->dev, "Hardware error: command enqueue failed\n");  in riscv_iommu_queue_send()
453  * IOMMU Command queue chapter 3.1
463 	ctrl = riscv_iommu_readl(queue->iommu, queue->qcr);  in riscv_iommu_cmdq_process()
466 		riscv_iommu_writel(queue->iommu, queue->qcr, ctrl);  in riscv_iommu_cmdq_process()
467 		dev_warn(queue->iommu->dev,  in riscv_iommu_cmdq_process()
479 	riscv_iommu_writel(queue->iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));  in riscv_iommu_cmdq_process()
484 /* Send command to the IOMMU command queue */
485 static void riscv_iommu_cmd_send(struct riscv_iommu_device *iommu,  in riscv_iommu_cmd_send()  argument
488 	riscv_iommu_queue_send(&iommu->cmdq, cmd, sizeof(*cmd));  in riscv_iommu_cmd_send()
492 static void riscv_iommu_cmd_sync(struct riscv_iommu_device *iommu,  in riscv_iommu_cmd_sync()  argument
499 	prod = riscv_iommu_queue_send(&iommu->cmdq, &cmd, sizeof(cmd));  in riscv_iommu_cmd_sync()
504 	if (riscv_iommu_queue_wait(&iommu->cmdq, prod, timeout_us))  in riscv_iommu_cmd_sync()
505 		dev_err_once(iommu->dev,  in riscv_iommu_cmd_sync()
510  * IOMMU Fault/Event queue chapter 3.2
513 static void riscv_iommu_fault(struct riscv_iommu_device *iommu,  in riscv_iommu_fault()  argument
521 		dev_warn_ratelimited(iommu->dev,  in riscv_iommu_fault()
530 	struct riscv_iommu_device *iommu = queue->iommu;  in riscv_iommu_fltq_process()  local
538 	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));  in riscv_iommu_fltq_process()
543 			riscv_iommu_fault(iommu, &events[Q_ITEM(queue, idx)]);  in riscv_iommu_fltq_process()
548 	ctrl = riscv_iommu_readl(iommu, queue->qcr);  in riscv_iommu_fltq_process()
550 		riscv_iommu_writel(iommu, queue->qcr, ctrl);  in riscv_iommu_fltq_process()
551 		dev_warn(iommu->dev,  in riscv_iommu_fltq_process()
562 static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,  in riscv_iommu_get_dc()  argument
565 	const bool base_format = !(iommu->caps & RISCV_IOMMU_CAPABILITIES_MSI_FLAT);  in riscv_iommu_get_dc()
573 	if (iommu->ddt_mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL ||  in riscv_iommu_get_dc()
574 	    iommu->ddt_mode > RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL)  in riscv_iommu_get_dc()
599 	depth = iommu->ddt_mode - RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL;  in riscv_iommu_get_dc()
604 	for (ddtp = iommu->ddt_root; depth-- > 0;) {  in riscv_iommu_get_dc()
623 			ptr = riscv_iommu_get_pages(iommu, 0);  in riscv_iommu_get_dc()
636 			riscv_iommu_free_pages(iommu, ptr);  in riscv_iommu_get_dc()
651  * This is best effort IOMMU translation shutdown flow.
652  * Disable IOMMU without waiting for hardware response.
654 void riscv_iommu_disable(struct riscv_iommu_device *iommu)  in riscv_iommu_disable()  argument
656 	riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,  in riscv_iommu_disable()
659 	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_CQCSR, 0);  in riscv_iommu_disable()
660 	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FQCSR, 0);  in riscv_iommu_disable()
661 	riscv_iommu_writel(iommu, RISCV_IOMMU_REG_PQCSR, 0);  in riscv_iommu_disable()
664 #define riscv_iommu_read_ddtp(iommu) ({ \  argument
666 	riscv_iommu_readq_timeout((iommu), RISCV_IOMMU_REG_DDTP, ddtp, \
671 static int riscv_iommu_iodir_alloc(struct riscv_iommu_device *iommu)  in riscv_iommu_iodir_alloc()  argument
676 	ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_alloc()
688 		riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,  in riscv_iommu_iodir_alloc()
690 		ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_alloc()
694 		iommu->ddt_phys = ppn_to_phys(ddtp);  in riscv_iommu_iodir_alloc()
695 		if (iommu->ddt_phys)  in riscv_iommu_iodir_alloc()
696 			iommu->ddt_root = devm_ioremap(iommu->dev,  in riscv_iommu_iodir_alloc()
697 						       iommu->ddt_phys, PAGE_SIZE);  in riscv_iommu_iodir_alloc()
698 		if (iommu->ddt_root)  in riscv_iommu_iodir_alloc()
699 			memset(iommu->ddt_root, 0, PAGE_SIZE);  in riscv_iommu_iodir_alloc()
702 	if (!iommu->ddt_root) {  in riscv_iommu_iodir_alloc()
703 		iommu->ddt_root = riscv_iommu_get_pages(iommu, 0);  in riscv_iommu_iodir_alloc()
704 		iommu->ddt_phys = __pa(iommu->ddt_root);  in riscv_iommu_iodir_alloc()
707 	if (!iommu->ddt_root)  in riscv_iommu_iodir_alloc()
716  * Accepted iommu->ddt_mode is updated on success.
718 static int riscv_iommu_iodir_set_mode(struct riscv_iommu_device *iommu,  in riscv_iommu_iodir_set_mode()  argument
721 	struct device *dev = iommu->dev;  in riscv_iommu_iodir_set_mode()
726 	ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_set_mode()
741 			rq_ddtp |= phys_to_ppn(iommu->ddt_phys);  in riscv_iommu_iodir_set_mode()
743 		riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP, rq_ddtp);  in riscv_iommu_iodir_set_mode()
744 		ddtp = riscv_iommu_read_ddtp(iommu);  in riscv_iommu_iodir_set_mode()
751 		/* Verify IOMMU hardware accepts new DDTP config. */  in riscv_iommu_iodir_set_mode()
765 		 * Mode field is WARL, an IOMMU may support a subset of  in riscv_iommu_iodir_set_mode()
779 		 * We tried all supported modes and IOMMU hardware failed to  in riscv_iommu_iodir_set_mode()
788 	iommu->ddt_mode = mode;  in riscv_iommu_iodir_set_mode()
794 	riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_iodir_set_mode()
798 	riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_iodir_set_mode()
801 	riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);  in riscv_iommu_iodir_set_mode()
806 /* This struct contains protection domain specific IOMMU driver data. */
821 /* Private IOMMU data for managed devices, dev_iommu_priv_* */
833  * Blocking and identity domains are not tracked here, as the IOMMU hardware
837  * The device pointer and IOMMU data remain stable in the bond struct after
838  * _probe_device() where it's attached to the managed IOMMU, up to the
851 	struct riscv_iommu_device *iommu = dev_to_iommu(dev);  in riscv_iommu_bond_link()  local
862 	 * managed IOMMU device.  in riscv_iommu_bond_link()
867 		if (dev_to_iommu(list_entry(bonds, struct riscv_iommu_bond, list)->dev) == iommu)  in riscv_iommu_bond_link()
881 	struct riscv_iommu_device *iommu = dev_to_iommu(dev);  in riscv_iommu_bond_unlink()  local
895 		else if (dev_to_iommu(bond->dev) == iommu)  in riscv_iommu_bond_unlink()
904 	 * If this was the last bond between this domain and the IOMMU  in riscv_iommu_bond_unlink()
910 		riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_bond_unlink()
912 		riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);  in riscv_iommu_bond_unlink()
919  * the hardware, when RISC-V IOMMU architecture specification update for
928 	struct riscv_iommu_device *iommu, *prev;  in riscv_iommu_iotlb_inval()  local
934 	 * For each IOMMU linked with this protection domain (via bonds->dev),  in riscv_iommu_iotlb_inval()
951 	 * be configured with already valid page table content. If an IOMMU is  in riscv_iommu_iotlb_inval()
961 		iommu = dev_to_iommu(bond->dev);  in riscv_iommu_iotlb_inval()
965 		 * to the IOMMU for the same PSCID, and with domain->bonds list  in riscv_iommu_iotlb_inval()
966 		 * arranged based on the device's IOMMU, it's sufficient to check  in riscv_iommu_iotlb_inval()
969 		if (iommu == prev)  in riscv_iommu_iotlb_inval()
977 				riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_iotlb_inval()
980 			riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_iotlb_inval()
982 		prev = iommu;  in riscv_iommu_iotlb_inval()
987 		iommu = dev_to_iommu(bond->dev);  in riscv_iommu_iotlb_inval()
988 		if (iommu == prev)  in riscv_iommu_iotlb_inval()
991 		riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);  in riscv_iommu_iotlb_inval()
992 		prev = iommu;  in riscv_iommu_iotlb_inval()
1006  * cached by the IOMMU hardware.
1011 static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,  in riscv_iommu_iodir_update()  argument
1022 		dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);  in riscv_iommu_iodir_update()
1032 		riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_iodir_update()
1037 		riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);  in riscv_iommu_iodir_update()
1044 		dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);  in riscv_iommu_iodir_update()
1057 		riscv_iommu_cmd_send(iommu, &cmd);  in riscv_iommu_iodir_update()
1060 	riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);  in riscv_iommu_iodir_update()
1305 static bool riscv_iommu_pt_supported(struct riscv_iommu_device *iommu, int pgd_mode)  in riscv_iommu_pt_supported()  argument
1309 		return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39;  in riscv_iommu_pt_supported()
1312 		return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48;  in riscv_iommu_pt_supported()
1315 		return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57;  in riscv_iommu_pt_supported()
1324 	struct riscv_iommu_device *iommu = dev_to_iommu(dev);  in riscv_iommu_attach_paging_domain()  local
1328 	if (!riscv_iommu_pt_supported(iommu, domain->pgd_mode))  in riscv_iommu_attach_paging_domain()
1339 	riscv_iommu_iodir_update(iommu, dev, fsc, ta);  in riscv_iommu_attach_paging_domain()
1359 	struct riscv_iommu_device *iommu;  in riscv_iommu_alloc_paging_domain()  local
1364 	iommu = dev_to_iommu(dev);  in riscv_iommu_alloc_paging_domain()
1365 	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57) {  in riscv_iommu_alloc_paging_domain()
1368 	} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48) {  in riscv_iommu_alloc_paging_domain()
1371 	} else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39) {  in riscv_iommu_alloc_paging_domain()
1385 	domain->numa_node = dev_to_node(iommu->dev);  in riscv_iommu_alloc_paging_domain()
1386 	domain->amo_enabled = !!(iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD);  in riscv_iommu_alloc_paging_domain()
1428 	struct riscv_iommu_device *iommu = dev_to_iommu(dev);  in riscv_iommu_attach_blocking_domain()  local
1432 	riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, 0);  in riscv_iommu_attach_blocking_domain()
1449 	struct riscv_iommu_device *iommu = dev_to_iommu(dev);  in riscv_iommu_attach_identity_domain()  local
1452 	riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, RISCV_IOMMU_PC_TA_V);  in riscv_iommu_attach_identity_domain()
1481 	struct riscv_iommu_device *iommu;  in riscv_iommu_probe_device()  local
1490 	iommu = dev_get_drvdata(fwspec->iommu_fwnode->dev);  in riscv_iommu_probe_device()
1491 	if (!iommu)  in riscv_iommu_probe_device()
1495 	 * IOMMU hardware operating in fail-over BARE mode will provide  in riscv_iommu_probe_device()
1498 	if (iommu->ddt_mode <= RISCV_IOMMU_DDTP_IOMMU_MODE_BARE)  in riscv_iommu_probe_device()
1509 	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD)  in riscv_iommu_probe_device()
1512 		dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);  in riscv_iommu_probe_device()
1518 			dev_warn(dev, "already attached to IOMMU device directory\n");  in riscv_iommu_probe_device()
1524 	return &iommu->iommu;  in riscv_iommu_probe_device()
1546 static int riscv_iommu_init_check(struct riscv_iommu_device *iommu)  in riscv_iommu_init_check()  argument
1551 	 * Make sure the IOMMU is switched off or in pass-through mode during  in riscv_iommu_init_check()
1555 	ddtp = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_DDTP);  in riscv_iommu_init_check()
1563 		riscv_iommu_disable(iommu);  in riscv_iommu_init_check()
1568 	    !!(iommu->fctl & RISCV_IOMMU_FCTL_BE)) {  in riscv_iommu_init_check()
1569 		if (!(iommu->caps & RISCV_IOMMU_CAPABILITIES_END))  in riscv_iommu_init_check()
1571 		riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL,  in riscv_iommu_init_check()
1572 				   iommu->fctl ^ RISCV_IOMMU_FCTL_BE);  in riscv_iommu_init_check()
1573 		iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);  in riscv_iommu_init_check()
1575 		    !!(iommu->fctl & RISCV_IOMMU_FCTL_BE))  in riscv_iommu_init_check()
1583 	if (!iommu->irqs_count)  in riscv_iommu_init_check()
1586 	iommu->icvec = FIELD_PREP(RISCV_IOMMU_ICVEC_FIV, 1 % iommu->irqs_count) |  in riscv_iommu_init_check()
1587 		       FIELD_PREP(RISCV_IOMMU_ICVEC_PIV, 2 % iommu->irqs_count) |  in riscv_iommu_init_check()
1588 		       FIELD_PREP(RISCV_IOMMU_ICVEC_PMIV, 3 % iommu->irqs_count);  in riscv_iommu_init_check()
1589 	riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_ICVEC, iommu->icvec);  in riscv_iommu_init_check()
1590 	iommu->icvec = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_ICVEC);  in riscv_iommu_init_check()
1591 	if (max(max(FIELD_GET(RISCV_IOMMU_ICVEC_CIV, iommu->icvec),  in riscv_iommu_init_check()
1592 		    FIELD_GET(RISCV_IOMMU_ICVEC_FIV, iommu->icvec)),  in riscv_iommu_init_check()
1593 		max(FIELD_GET(RISCV_IOMMU_ICVEC_PIV, iommu->icvec),  in riscv_iommu_init_check()
1594 		    FIELD_GET(RISCV_IOMMU_ICVEC_PMIV, iommu->icvec))) >= iommu->irqs_count)  in riscv_iommu_init_check()
1600 void riscv_iommu_remove(struct riscv_iommu_device *iommu)  in riscv_iommu_remove()  argument
1602 	iommu_device_unregister(&iommu->iommu);  in riscv_iommu_remove()
1603 	iommu_device_sysfs_remove(&iommu->iommu);  in riscv_iommu_remove()
1604 	riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);  in riscv_iommu_remove()
1605 	riscv_iommu_queue_disable(&iommu->cmdq);  in riscv_iommu_remove()
1606 	riscv_iommu_queue_disable(&iommu->fltq);  in riscv_iommu_remove()
1609 int riscv_iommu_init(struct riscv_iommu_device *iommu)  in riscv_iommu_init()  argument
1613 	RISCV_IOMMU_QUEUE_INIT(&iommu->cmdq, CQ);  in riscv_iommu_init()
1614 	RISCV_IOMMU_QUEUE_INIT(&iommu->fltq, FQ);  in riscv_iommu_init()
1616 	rc = riscv_iommu_init_check(iommu);  in riscv_iommu_init()
1618 		return dev_err_probe(iommu->dev, rc, "unexpected device state\n");  in riscv_iommu_init()
1620 	rc = riscv_iommu_iodir_alloc(iommu);  in riscv_iommu_init()
1624 	rc = riscv_iommu_queue_alloc(iommu, &iommu->cmdq,  in riscv_iommu_init()
1629 	rc = riscv_iommu_queue_alloc(iommu, &iommu->fltq,  in riscv_iommu_init()
1634 	rc = riscv_iommu_queue_enable(iommu, &iommu->cmdq, riscv_iommu_cmdq_process);  in riscv_iommu_init()
1638 	rc = riscv_iommu_queue_enable(iommu, &iommu->fltq, riscv_iommu_fltq_process);  in riscv_iommu_init()
1642 	rc = riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_MAX);  in riscv_iommu_init()
1646 	rc = iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "riscv-iommu@%s",  in riscv_iommu_init()
1647 				    dev_name(iommu->dev));  in riscv_iommu_init()
1649 		dev_err_probe(iommu->dev, rc, "cannot register sysfs interface\n");  in riscv_iommu_init()
1653 	rc = iommu_device_register(&iommu->iommu, &riscv_iommu_ops, iommu->dev);  in riscv_iommu_init()
1655 		dev_err_probe(iommu->dev, rc, "cannot register iommu interface\n");  in riscv_iommu_init()
1662 	iommu_device_sysfs_remove(&iommu->iommu);  in riscv_iommu_init()
1664 	riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);  in riscv_iommu_init()
1666 	riscv_iommu_queue_disable(&iommu->fltq);  in riscv_iommu_init()
1667 	riscv_iommu_queue_disable(&iommu->cmdq);  in riscv_iommu_init()