28  * Helper function to get the first pte of a large mapping
30 static u64 *first_pte_l7(u64 *pte, unsigned long *page_size,  in first_pte_l7()  argument
36 	pg_size  = PTE_PAGE_SIZE(*pte);  in first_pte_l7()
39 	fpte     = (u64 *)(((unsigned long)pte) & pte_mask);  in first_pte_l7()
70 		/* PTE present? */  in free_pt_lvl()
74 		/* Large PTE? */  in free_pt_lvl()
129 	u64 *pte;  in increase_address_space()  local
131 	pte = iommu_alloc_page_node(cfg->amd.nid, gfp);  in increase_address_space()
132 	if (!pte)  in increase_address_space()
145 	*pte = PM_LEVEL_PDE(pgtable->mode, iommu_virt_to_phys(pgtable->root));  in increase_address_space()
147 	pgtable->root  = pte;  in increase_address_space()
151 	pte = NULL;  in increase_address_space()
156 	iommu_free_page(pte);  in increase_address_space()
171 	u64 *pte, *page;  in alloc_pte()  local
188 	pte     = &pgtable->root[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
196 		__pte     = *pte;  in alloc_pte()
208 			lpte = first_pte_l7(pte, NULL, &count);  in alloc_pte()
230 			/* pte could have been changed somewhere. */  in alloc_pte()
231 			if (!try_cmpxchg64(pte, &__pte, __npte))  in alloc_pte()
245 		pte = IOMMU_PTE_PAGE(__pte);  in alloc_pte()
248 			*pte_page = pte;  in alloc_pte()
250 		pte = &pte[PM_LEVEL_INDEX(level, address)];  in alloc_pte()
253 	return pte;  in alloc_pte()
257  * This function checks if there is a PTE for a given dma address. If
265 	u64 *pte;  in fetch_pte()  local
273 	pte	   = &pgtable->root[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
279 		if (!IOMMU_PTE_PRESENT(*pte))  in fetch_pte()
282 		/* Large PTE */  in fetch_pte()
283 		if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL ||  in fetch_pte()
284 		    PM_PTE_LEVEL(*pte) == PAGE_MODE_NONE)  in fetch_pte()
288 		if (PM_PTE_LEVEL(*pte) != level)  in fetch_pte()
294 		pte	   = IOMMU_PTE_PAGE(*pte);  in fetch_pte()
295 		pte	   = &pte[PM_LEVEL_INDEX(level, address)];  in fetch_pte()
303 	if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL)  in fetch_pte()
304 		pte = first_pte_l7(pte, page_size, NULL);  in fetch_pte()
306 	return pte;  in fetch_pte()
309 static void free_clear_pte(u64 *pte, u64 pteval, struct list_head *freelist)  in free_clear_pte()  argument
314 	while (!try_cmpxchg64(pte, &pteval, 0))  in free_clear_pte()
315 		pr_warn("AMD-Vi: IOMMU pte changed since we read it\n");  in free_clear_pte()
340 	u64 __pte, *pte;  in iommu_v1_map_pages()  local
354 		pte   = alloc_pte(pgtable, iova, pgsize, NULL, gfp, &updated);  in iommu_v1_map_pages()
357 		if (!pte)  in iommu_v1_map_pages()
361 			free_clear_pte(&pte[i], pte[i], &freelist);  in iommu_v1_map_pages()
378 			pte[i] = __pte;  in iommu_v1_map_pages()
418 	u64 *pte;  in iommu_v1_unmap_pages()  local
426 		pte = fetch_pte(pgtable, iova, &unmap_size);  in iommu_v1_unmap_pages()
427 		if (pte) {  in iommu_v1_unmap_pages()
432 				pte[i] = 0ULL;  in iommu_v1_unmap_pages()
448 	u64 *pte, __pte;  in iommu_v1_iova_to_phys()  local
450 	pte = fetch_pte(pgtable, iova, &pte_pgsize);  in iommu_v1_iova_to_phys()
452 	if (!pte || !IOMMU_PTE_PRESENT(*pte))  in iommu_v1_iova_to_phys()
456 	__pte	    = __sme_clr(*pte & PM_ADDR_MASK);  in iommu_v1_iova_to_phys()
505 		u64 *ptep, pte;  in iommu_v1_read_and_clear_dirty()  local
509 			pte = READ_ONCE(*ptep);  in iommu_v1_read_and_clear_dirty()
510 		if (!ptep || !IOMMU_PTE_PRESENT(pte)) {  in iommu_v1_read_and_clear_dirty()