44 static inline bool is_large_pte(u64 pte)  in is_large_pte()  argument
46 	return (pte & IOMMU_PAGE_PSE);  in is_large_pte()
59 static inline void *get_pgtable_pte(u64 pte)  in get_pgtable_pte()  argument
61 	return iommu_phys_to_virt(pte & PM_ADDR_MASK);  in get_pgtable_pte()
66 	u64 pte;  in set_pte_attr()  local
68 	pte = __sme_set(paddr & PM_ADDR_MASK);  in set_pte_attr()
69 	pte |= IOMMU_PAGE_PRESENT | IOMMU_PAGE_USER;  in set_pte_attr()
70 	pte |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY;  in set_pte_attr()
73 		pte |= IOMMU_PAGE_RW;  in set_pte_attr()
77 		pte |= IOMMU_PAGE_PSE;  in set_pte_attr()
79 	return pte;  in set_pte_attr()
109 		/* PTE present? */  in free_pgtable()
134 	u64 *pte, *page;  in v2_alloc_pte()  local
139 	pte = &pgd[PM_LEVEL_INDEX(level, iova)];  in v2_alloc_pte()
145 		__pte = *pte;  in v2_alloc_pte()
148 			/* Unmap large pte */  in v2_alloc_pte()
149 			cmpxchg64(pte, *pte, 0ULL);  in v2_alloc_pte()
160 			/* pte could have been changed somewhere. */  in v2_alloc_pte()
161 			if (!try_cmpxchg64(pte, &__pte, __npte))  in v2_alloc_pte()
170 		pte = get_pgtable_pte(__pte);  in v2_alloc_pte()
171 		pte = &pte[PM_LEVEL_INDEX(level, iova)];  in v2_alloc_pte()
174 	/* Tear down existing pte entries */  in v2_alloc_pte()
175 	if (IOMMU_PTE_PRESENT(*pte)) {  in v2_alloc_pte()
179 		__pte = get_pgtable_pte(*pte);  in v2_alloc_pte()
180 		cmpxchg64(pte, *pte, 0ULL);  in v2_alloc_pte()
187 	return pte;  in v2_alloc_pte()
191  * This function checks if there is a PTE for a given dma address.
197 	u64 *pte;  in fetch_pte()  local
201 	pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)];  in fetch_pte()
207 		if (!IOMMU_PTE_PRESENT(*pte))  in fetch_pte()
211 		pte = get_pgtable_pte(*pte);  in fetch_pte()
212 		pte = &pte[PM_LEVEL_INDEX(level - 1, iova)];  in fetch_pte()
215 		if (is_large_pte(*pte)) {  in fetch_pte()
221 				return NULL;	/* Wrongly set PSE bit in PTE */  in fetch_pte()
229 	return pte;  in fetch_pte()
238 	u64 *pte;  in iommu_v2_map_pages()  local
254 		pte = v2_alloc_pte(cfg->amd.nid, pgtable->pgd,  in iommu_v2_map_pages()
256 		if (!pte) {  in iommu_v2_map_pages()
261 		*pte = set_pte_attr(paddr, map_size, prot);  in iommu_v2_map_pages()
294 	u64 *pte;  in iommu_v2_unmap_pages()  local
300 		pte = fetch_pte(pgtable, iova, &unmap_size);  in iommu_v2_unmap_pages()
301 		if (!pte)  in iommu_v2_unmap_pages()
304 		*pte = 0ULL;  in iommu_v2_unmap_pages()
317 	u64 *pte, __pte;  in iommu_v2_iova_to_phys()  local
319 	pte = fetch_pte(pgtable, iova, &pte_pgsize);  in iommu_v2_iova_to_phys()
320 	if (!pte || !IOMMU_PTE_PRESENT(*pte))  in iommu_v2_iova_to_phys()
324 	__pte = __sme_clr(*pte & PM_ADDR_MASK);  in iommu_v2_iova_to_phys()