24 #include "iommu.h"
26 #include "../iommu-pages.h"
29 	struct intel_iommu *iommu;  member
36 	struct intel_iommu *iommu;  member
43 	struct intel_iommu *iommu;  member
71  *	->iommu->register_lock
80 static void iommu_disable_irq_remapping(struct intel_iommu *iommu);
84 static bool ir_pre_enabled(struct intel_iommu *iommu)  in ir_pre_enabled()  argument
86 	return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED);  in ir_pre_enabled()
89 static void clear_ir_pre_enabled(struct intel_iommu *iommu)  in clear_ir_pre_enabled()  argument
91 	iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in clear_ir_pre_enabled()
94 static void init_ir_status(struct intel_iommu *iommu)  in init_ir_status()  argument
98 	gsts = readl(iommu->reg + DMAR_GSTS_REG);  in init_ir_status()
100 		iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED;  in init_ir_status()
103 static int alloc_irte(struct intel_iommu *iommu,  in alloc_irte()  argument
106 	struct ir_table *table = iommu->ir_table;  in alloc_irte()
119 	if (mask > ecap_max_handle_mask(iommu->ecap)) {  in alloc_irte()
122 		       ecap_max_handle_mask(iommu->ecap));  in alloc_irte()
130 		pr_warn("IR%d: can't allocate an IRTE\n", iommu->seq_id);  in alloc_irte()
132 		irq_iommu->iommu = iommu;  in alloc_irte()
142 static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)  in qi_flush_iec()  argument
152 	return qi_submit_sync(iommu, &desc, 1, 0);  in qi_flush_iec()
158 	struct intel_iommu *iommu;  in modify_irte()  local
168 	iommu = irq_iommu->iommu;  in modify_irte()
171 	irte = &iommu->ir_table->base[index];  in modify_irte()
186 	__iommu_flush_cache(iommu, irte, sizeof(*irte));  in modify_irte()
188 	rc = qi_flush_iec(iommu, index, 0);  in modify_irte()
200 		if (ir_hpet[i].id == hpet_id && ir_hpet[i].iommu)  in map_hpet_to_iommu()
201 			return ir_hpet[i].iommu;  in map_hpet_to_iommu()
211 		if (ir_ioapic[i].id == apic && ir_ioapic[i].iommu)  in map_ioapic_to_iommu()
212 			return ir_ioapic[i].iommu;  in map_ioapic_to_iommu()
221 	return drhd ? drhd->iommu->ir_domain : NULL;  in map_dev_to_ir()
227 	struct intel_iommu *iommu;  in clear_entries()  local
233 	iommu = irq_iommu->iommu;  in clear_entries()
236 	start = iommu->ir_table->base + index;  in clear_entries()
243 	bitmap_release_region(iommu->ir_table->bitmap, index,  in clear_entries()
246 	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);  in clear_entries()
305 		if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) {  in set_ioapic_sid()
330 		if (ir_hpet[i].iommu && ir_hpet[i].id == id) {  in set_hpet_sid()
415 static int iommu_load_old_irte(struct intel_iommu *iommu)  in iommu_load_old_irte()  argument
424 	irta = dmar_readq(iommu->reg + DMAR_IRTA_REG);  in iommu_load_old_irte()
438 	memcpy(iommu->ir_table->base, old_ir_table, size);  in iommu_load_old_irte()
440 	__iommu_flush_cache(iommu, iommu->ir_table->base, size);  in iommu_load_old_irte()
447 		if (iommu->ir_table->base[i].present)  in iommu_load_old_irte()
448 			bitmap_set(iommu->ir_table->bitmap, i, 1);  in iommu_load_old_irte()
457 static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode)  in iommu_set_irq_remapping()  argument
463 	addr = virt_to_phys((void *)iommu->ir_table->base);  in iommu_set_irq_remapping()
465 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
467 	dmar_writeq(iommu->reg + DMAR_IRTA_REG,  in iommu_set_irq_remapping()
471 	writel(iommu->gcmd | DMA_GCMD_SIRTP, iommu->reg + DMAR_GCMD_REG);  in iommu_set_irq_remapping()
473 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_set_irq_remapping()
475 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_set_irq_remapping()
481 	if (!cap_esirtps(iommu->cap))  in iommu_set_irq_remapping()
482 		qi_global_iec(iommu);  in iommu_set_irq_remapping()
485 static void iommu_enable_irq_remapping(struct intel_iommu *iommu)  in iommu_enable_irq_remapping()  argument
490 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
493 	iommu->gcmd |= DMA_GCMD_IRE;  in iommu_enable_irq_remapping()
494 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
495 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
500 		iommu->gcmd &= ~DMA_GCMD_CFI;  in iommu_enable_irq_remapping()
501 		writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_enable_irq_remapping()
502 		IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_enable_irq_remapping()
516 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_enable_irq_remapping()
519 static int intel_setup_irq_remapping(struct intel_iommu *iommu)  in intel_setup_irq_remapping()  argument
526 	if (iommu->ir_table)  in intel_setup_irq_remapping()
533 	ir_table_base = iommu_alloc_pages_node(iommu->node, GFP_KERNEL,  in intel_setup_irq_remapping()
537 		       iommu->seq_id, INTR_REMAP_PAGE_ORDER);  in intel_setup_irq_remapping()
543 		pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id);  in intel_setup_irq_remapping()
547 	fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);  in intel_setup_irq_remapping()
551 	iommu->ir_domain =  in intel_setup_irq_remapping()
555 					    iommu);  in intel_setup_irq_remapping()
556 	if (!iommu->ir_domain) {  in intel_setup_irq_remapping()
557 		pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);  in intel_setup_irq_remapping()
561 	irq_domain_update_bus_token(iommu->ir_domain,  DOMAIN_BUS_DMAR);  in intel_setup_irq_remapping()
562 	iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT |  in intel_setup_irq_remapping()
564 	iommu->ir_domain->msi_parent_ops = &dmar_msi_parent_ops;  in intel_setup_irq_remapping()
568 	iommu->ir_table = ir_table;  in intel_setup_irq_remapping()
574 	if (!iommu->qi) {  in intel_setup_irq_remapping()
578 		dmar_fault(-1, iommu);  in intel_setup_irq_remapping()
579 		dmar_disable_qi(iommu);  in intel_setup_irq_remapping()
581 		if (dmar_enable_qi(iommu)) {  in intel_setup_irq_remapping()
587 	init_ir_status(iommu);  in intel_setup_irq_remapping()
589 	if (ir_pre_enabled(iommu)) {  in intel_setup_irq_remapping()
592 				     iommu->name);  in intel_setup_irq_remapping()
593 			clear_ir_pre_enabled(iommu);  in intel_setup_irq_remapping()
594 			iommu_disable_irq_remapping(iommu);  in intel_setup_irq_remapping()
595 		} else if (iommu_load_old_irte(iommu))  in intel_setup_irq_remapping()
597 			       iommu->name);  in intel_setup_irq_remapping()
600 				iommu->name);  in intel_setup_irq_remapping()
603 	iommu_set_irq_remapping(iommu, eim_mode);  in intel_setup_irq_remapping()
608 	irq_domain_remove(iommu->ir_domain);  in intel_setup_irq_remapping()
609 	iommu->ir_domain = NULL;  in intel_setup_irq_remapping()
619 	iommu->ir_table  = NULL;  in intel_setup_irq_remapping()
624 static void intel_teardown_irq_remapping(struct intel_iommu *iommu)  in intel_teardown_irq_remapping()  argument
628 	if (iommu && iommu->ir_table) {  in intel_teardown_irq_remapping()
629 		if (iommu->ir_domain) {  in intel_teardown_irq_remapping()
630 			fn = iommu->ir_domain->fwnode;  in intel_teardown_irq_remapping()
632 			irq_domain_remove(iommu->ir_domain);  in intel_teardown_irq_remapping()
634 			iommu->ir_domain = NULL;  in intel_teardown_irq_remapping()
636 		iommu_free_pages(iommu->ir_table->base, INTR_REMAP_PAGE_ORDER);  in intel_teardown_irq_remapping()
637 		bitmap_free(iommu->ir_table->bitmap);  in intel_teardown_irq_remapping()
638 		kfree(iommu->ir_table);  in intel_teardown_irq_remapping()
639 		iommu->ir_table = NULL;  in intel_teardown_irq_remapping()
646 static void iommu_disable_irq_remapping(struct intel_iommu *iommu)  in iommu_disable_irq_remapping()  argument
651 	if (!ecap_ir_support(iommu->ecap))  in iommu_disable_irq_remapping()
658 	if (!cap_esirtps(iommu->cap))  in iommu_disable_irq_remapping()
659 		qi_global_iec(iommu);  in iommu_disable_irq_remapping()
661 	raw_spin_lock_irqsave(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
663 	sts = readl(iommu->reg + DMAR_GSTS_REG);  in iommu_disable_irq_remapping()
667 	iommu->gcmd &= ~DMA_GCMD_IRE;  in iommu_disable_irq_remapping()
668 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);  in iommu_disable_irq_remapping()
670 	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,  in iommu_disable_irq_remapping()
674 	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);  in iommu_disable_irq_remapping()
689 	struct intel_iommu *iommu;  in intel_cleanup_irq_remapping()  local
691 	for_each_iommu(iommu, drhd) {  in intel_cleanup_irq_remapping()
692 		if (ecap_ir_support(iommu->ecap)) {  in intel_cleanup_irq_remapping()
693 			iommu_disable_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
694 			intel_teardown_irq_remapping(iommu);  in intel_cleanup_irq_remapping()
705 	struct intel_iommu *iommu;  in intel_prepare_irq_remapping()  local
730 	for_each_iommu(iommu, drhd)  in intel_prepare_irq_remapping()
731 		if (!ecap_ir_support(iommu->ecap))  in intel_prepare_irq_remapping()
743 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
744 		if (eim && !ecap_eim_support(iommu->ecap)) {  in intel_prepare_irq_remapping()
745 			pr_info("%s does not support EIM\n", iommu->name);  in intel_prepare_irq_remapping()
755 	for_each_iommu(iommu, drhd) {  in intel_prepare_irq_remapping()
756 		if (intel_setup_irq_remapping(iommu)) {  in intel_prepare_irq_remapping()
758 			       iommu->name);  in intel_prepare_irq_remapping()
776 	struct intel_iommu *iommu;  in set_irq_posting_cap()  local
790 		for_each_iommu(iommu, drhd)  in set_irq_posting_cap()
791 			if (!cap_pi_support(iommu->cap)) {  in set_irq_posting_cap()
802 	struct intel_iommu *iommu;  in intel_enable_irq_remapping()  local
808 	for_each_iommu(iommu, drhd) {  in intel_enable_irq_remapping()
809 		if (!ir_pre_enabled(iommu))  in intel_enable_irq_remapping()
810 			iommu_enable_irq_remapping(iommu);  in intel_enable_irq_remapping()
831 				   struct intel_iommu *iommu,  in ir_parse_one_hpet_scope()  argument
854 		if (ir_hpet[count].iommu == iommu &&  in ir_parse_one_hpet_scope()
857 		else if (ir_hpet[count].iommu == NULL && free == -1)  in ir_parse_one_hpet_scope()
865 	ir_hpet[free].iommu = iommu;  in ir_parse_one_hpet_scope()
876 				     struct intel_iommu *iommu,  in ir_parse_one_ioapic_scope()  argument
899 		if (ir_ioapic[count].iommu == iommu &&  in ir_parse_one_ioapic_scope()
902 		else if (ir_ioapic[count].iommu == NULL && free == -1)  in ir_parse_one_ioapic_scope()
912 	ir_ioapic[free].iommu = iommu;  in ir_parse_one_ioapic_scope()
914 	pr_info("IOAPIC id %d under DRHD base  0x%Lx IOMMU %d\n",  in ir_parse_one_ioapic_scope()
915 		scope->enumeration_id, drhd->address, iommu->seq_id);  in ir_parse_one_ioapic_scope()
921 				      struct intel_iommu *iommu)  in ir_parse_ioapic_hpet_scope()  argument
935 			ret = ir_parse_one_ioapic_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
937 			ret = ir_parse_one_hpet_scope(scope, iommu, drhd);  in ir_parse_ioapic_hpet_scope()
944 static void ir_remove_ioapic_hpet_scope(struct intel_iommu *iommu)  in ir_remove_ioapic_hpet_scope()  argument
949 		if (ir_hpet[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
950 			ir_hpet[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
953 		if (ir_ioapic[i].iommu == iommu)  in ir_remove_ioapic_hpet_scope()
954 			ir_ioapic[i].iommu = NULL;  in ir_remove_ioapic_hpet_scope()
964 	struct intel_iommu *iommu;  in parse_ioapics_under_ir()  local
968 	for_each_iommu(iommu, drhd) {  in parse_ioapics_under_ir()
971 		if (!ecap_ir_support(iommu->ecap))  in parse_ioapics_under_ir()
974 		ret = ir_parse_ioapic_hpet_scope(drhd->hdr, iommu);  in parse_ioapics_under_ir()
987 			pr_err(FW_BUG "ioapic %d has no mapping iommu, "  in parse_ioapics_under_ir()
1015 	struct intel_iommu *iommu = NULL;  in disable_irq_remapping()  local
1020 	for_each_iommu(iommu, drhd) {  in disable_irq_remapping()
1021 		if (!ecap_ir_support(iommu->ecap))  in disable_irq_remapping()
1024 		iommu_disable_irq_remapping(iommu);  in disable_irq_remapping()
1038 	struct intel_iommu *iommu = NULL;  in reenable_irq_remapping()  local
1040 	for_each_iommu(iommu, drhd)  in reenable_irq_remapping()
1041 		if (iommu->qi)  in reenable_irq_remapping()
1042 			dmar_reenable_qi(iommu);  in reenable_irq_remapping()
1047 	for_each_iommu(iommu, drhd) {  in reenable_irq_remapping()
1048 		if (!ecap_ir_support(iommu->ecap))  in reenable_irq_remapping()
1051 		/* Set up interrupt remapping for iommu.*/  in reenable_irq_remapping()
1052 		iommu_set_irq_remapping(iommu, eim);  in reenable_irq_remapping()
1053 		iommu_enable_irq_remapping(iommu);  in reenable_irq_remapping()
1414 	struct intel_iommu *iommu = domain->host_data;  in intel_irq_remapping_alloc()  local
1421 	if (!info || !iommu)  in intel_irq_remapping_alloc()
1435 	index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs);  in intel_irq_remapping_alloc()
1513 	struct intel_iommu *iommu = NULL;  in intel_irq_remapping_select()  local
1516 		iommu = map_ioapic_to_iommu(fwspec->param[0]);  in intel_irq_remapping_select()
1518 		iommu = map_hpet_to_iommu(fwspec->param[0]);  in intel_irq_remapping_select()
1520 	return iommu && d == iommu->ir_domain;  in intel_irq_remapping_select()
1540 static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu)  in dmar_ir_add()  argument
1545 	if (eim && !ecap_eim_support(iommu->ecap)) {  in dmar_ir_add()
1547 			iommu->reg_phys, iommu->ecap);  in dmar_ir_add()
1551 	if (ir_parse_ioapic_hpet_scope(dmaru->hdr, iommu)) {  in dmar_ir_add()
1553 			iommu->reg_phys);  in dmar_ir_add()
1557 	/* TODO: check all IOAPICs are covered by IOMMU */  in dmar_ir_add()
1560 	ret = intel_setup_irq_remapping(iommu);  in dmar_ir_add()
1563 		       iommu->name);  in dmar_ir_add()
1564 		intel_teardown_irq_remapping(iommu);  in dmar_ir_add()
1565 		ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_add()
1567 		iommu_enable_irq_remapping(iommu);  in dmar_ir_add()
1576 	struct intel_iommu *iommu = dmaru->iommu;  in dmar_ir_hotplug()  local
1580 	if (iommu == NULL)  in dmar_ir_hotplug()
1582 	if (!ecap_ir_support(iommu->ecap))  in dmar_ir_hotplug()
1585 	    !cap_pi_support(iommu->cap))  in dmar_ir_hotplug()
1589 		if (!iommu->ir_table)  in dmar_ir_hotplug()
1590 			ret = dmar_ir_add(dmaru, iommu);  in dmar_ir_hotplug()
1592 		if (iommu->ir_table) {  in dmar_ir_hotplug()
1593 			if (!bitmap_empty(iommu->ir_table->bitmap,  in dmar_ir_hotplug()
1597 				iommu_disable_irq_remapping(iommu);  in dmar_ir_hotplug()
1598 				intel_teardown_irq_remapping(iommu);  in dmar_ir_hotplug()
1599 				ir_remove_ioapic_hpet_scope(iommu);  in dmar_ir_hotplug()