194  * index is the number of PAGE_SIZE units from the start of the area's
195  * iopt_pages. If the iova is sub page-size then the area has an iova that
198 static unsigned long iopt_area_index_to_iova(struct iopt_area *area,  in iopt_area_index_to_iova()  argument
202 		WARN_ON(index < iopt_area_index(area) ||  in iopt_area_index_to_iova()
203 			index > iopt_area_last_index(area));  in iopt_area_index_to_iova()
204 	index -= iopt_area_index(area);  in iopt_area_index_to_iova()
206 		return iopt_area_iova(area);  in iopt_area_index_to_iova()
207 	return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE;  in iopt_area_index_to_iova()
210 static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area,  in iopt_area_index_to_iova_last()  argument
214 		WARN_ON(index < iopt_area_index(area) ||  in iopt_area_index_to_iova_last()
215 			index > iopt_area_last_index(area));  in iopt_area_index_to_iova_last()
216 	if (index == iopt_area_last_index(area))  in iopt_area_index_to_iova_last()
217 		return iopt_area_last_iova(area);  in iopt_area_index_to_iova_last()
218 	return iopt_area_iova(area) - area->page_offset +  in iopt_area_index_to_iova_last()
219 	       (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1;  in iopt_area_index_to_iova_last()
238 static void iopt_area_unmap_domain_range(struct iopt_area *area,  in iopt_area_unmap_domain_range()  argument
243 	unsigned long start_iova = iopt_area_index_to_iova(area, start_index);  in iopt_area_unmap_domain_range()
246 			   iopt_area_index_to_iova_last(area, last_index) -  in iopt_area_unmap_domain_range()
392 			      struct iopt_area *area, unsigned long start_index,  in batch_from_domain()  argument
399 	iova = iopt_area_index_to_iova(area, start_index);  in batch_from_domain()
400 	if (start_index == iopt_area_index(area))  in batch_from_domain()
401 		page_offset = area->page_offset;  in batch_from_domain()
418 					   struct iopt_area *area,  in raw_pages_from_domain()  argument
427 	iova = iopt_area_index_to_iova(area, start_index);  in raw_pages_from_domain()
428 	if (start_index == iopt_area_index(area))  in raw_pages_from_domain()
429 		page_offset = area->page_offset;  in raw_pages_from_domain()
443 				       struct iopt_area *area,  in batch_from_domain_continue()  argument
450 	batch_from_domain(batch, domain, area, start_index, last_index);  in batch_from_domain_continue()
456  * mode permits splitting a mapped area up, and then one of the splits is
491 			   struct iopt_area *area, unsigned long start_index)  in batch_to_domain()  argument
493 	bool disable_large_pages = area->iopt->disable_large_pages;  in batch_to_domain()
494 	unsigned long last_iova = iopt_area_last_iova(area);  in batch_to_domain()
503 	if (start_index == iopt_area_index(area))  in batch_to_domain()
504 		page_offset = area->page_offset;  in batch_to_domain()
506 		iopt_area_index_to_iova(area, start_index);  in batch_to_domain()
515 				next_iova - iova, area->iommu_prot);  in batch_to_domain()
519 				       next_iova - iova, area->iommu_prot,  in batch_to_domain()
1061  * - The iommu_domain under an area
1113 	struct iopt_area *area;  in pfn_reader_fill_span()  local
1130 		 * and we'll find another area.  in pfn_reader_fill_span()
1132 		area = iopt_pages_find_domain_area(pfns->pages, start_index);  in pfn_reader_fill_span()
1133 		if (WARN_ON(!area))  in pfn_reader_fill_span()
1138 			&pfns->batch, area->storage_domain, area, start_index,  in pfn_reader_fill_span()
1139 			min(iopt_area_last_index(area), span->last_used));  in pfn_reader_fill_span()
1382 iopt_area_unpin_domain(struct pfn_batch *batch, struct iopt_area *area,  in iopt_area_unpin_domain()  argument
1400 			batch_from_domain(batch, domain, area, start,  in iopt_area_unpin_domain()
1421 			batch_from_domain_continue(batch, domain, area,  in iopt_area_unpin_domain()
1427 				area, domain, *unmapped_end_index,  in iopt_area_unpin_domain()
1442 static void __iopt_area_unfill_domain(struct iopt_area *area,  in __iopt_area_unfill_domain()  argument
1448 	unsigned long start_index = iopt_area_index(area);  in __iopt_area_unfill_domain()
1478 		iopt_area_unpin_domain(&batch, area, pages, domain,  in __iopt_area_unfill_domain()
1487 		iopt_area_unmap_domain_range(area, domain, unmapped_end_index,  in __iopt_area_unfill_domain()
1494 static void iopt_area_unfill_partial_domain(struct iopt_area *area,  in iopt_area_unfill_partial_domain()  argument
1499 	if (end_index != iopt_area_index(area))  in iopt_area_unfill_partial_domain()
1500 		__iopt_area_unfill_domain(area, pages, domain, end_index - 1);  in iopt_area_unfill_partial_domain()
1505  * @area: The IOVA range to unmap
1511 void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain)  in iopt_area_unmap_domain()  argument
1513 	iommu_unmap_nofail(domain, iopt_area_iova(area),  in iopt_area_unmap_domain()
1514 			   iopt_area_length(area));  in iopt_area_unmap_domain()
1519  * @area: IOVA area to use
1520  * @pages: page supplier for the area (area->pages is NULL)
1527 void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages,  in iopt_area_unfill_domain()  argument
1530 	__iopt_area_unfill_domain(area, pages, domain,  in iopt_area_unfill_domain()
1531 				  iopt_area_last_index(area));  in iopt_area_unfill_domain()
1535  * iopt_area_fill_domain() - Map PFNs from the area into a domain
1536  * @area: IOVA area to use
1539  * Read the pfns from the area's underlying iopt_pages and map them into the
1542 int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain)  in iopt_area_fill_domain()  argument
1548 	lockdep_assert_held(&area->pages->mutex);  in iopt_area_fill_domain()
1550 	rc = pfn_reader_first(&pfns, area->pages, iopt_area_index(area),  in iopt_area_fill_domain()
1551 			      iopt_area_last_index(area));  in iopt_area_fill_domain()
1557 		rc = batch_to_domain(&pfns.batch, domain, area,  in iopt_area_fill_domain()
1575 	iopt_area_unfill_partial_domain(area, area->pages, domain,  in iopt_area_fill_domain()
1583  * iopt_area_fill_domains() - Install PFNs into the area's domains
1584  * @area: The area to act on
1585  * @pages: The pages associated with the area (area->pages is NULL)
1587  * Called during area creation. The area is freshly created and not inserted in
1589  * area's io_pagetable and the area is installed in the domains_itree.
1593 int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages)  in iopt_area_fill_domains()  argument
1603 	lockdep_assert_held(&area->iopt->domains_rwsem);  in iopt_area_fill_domains()
1605 	if (xa_empty(&area->iopt->domains))  in iopt_area_fill_domains()
1609 	rc = pfn_reader_first(&pfns, pages, iopt_area_index(area),  in iopt_area_fill_domains()
1610 			      iopt_area_last_index(area));  in iopt_area_fill_domains()
1617 		xa_for_each(&area->iopt->domains, index, domain) {  in iopt_area_fill_domains()
1618 			rc = batch_to_domain(&pfns.batch, domain, area,  in iopt_area_fill_domains()
1633 	area->storage_domain = xa_load(&area->iopt->domains, 0);  in iopt_area_fill_domains()
1634 	interval_tree_insert(&area->pages_node, &pages->domains_itree);  in iopt_area_fill_domains()
1639 	xa_for_each(&area->iopt->domains, unmap_index, domain) {  in iopt_area_fill_domains()
1648 		 * The area is not yet part of the domains_itree so we have to  in iopt_area_fill_domains()
1652 		if (unmap_index != area->iopt->next_domain_id - 1) {  in iopt_area_fill_domains()
1653 			if (end_index != iopt_area_index(area))  in iopt_area_fill_domains()
1655 					area, domain, iopt_area_index(area),  in iopt_area_fill_domains()
1658 			iopt_area_unfill_partial_domain(area, pages, domain,  in iopt_area_fill_domains()
1670  * iopt_area_unfill_domains() - unmap PFNs from the area's domains
1671  * @area: The area to act on
1672  * @pages: The pages associated with the area (area->pages is NULL)
1674  * Called during area destruction. This unmaps the iova's covered by all the
1675  * area's domains and releases the PFNs.
1677 void iopt_area_unfill_domains(struct iopt_area *area, struct iopt_pages *pages)  in iopt_area_unfill_domains()  argument
1679 	struct io_pagetable *iopt = area->iopt;  in iopt_area_unfill_domains()
1686 	if (!area->storage_domain)  in iopt_area_unfill_domains()
1690 		if (domain != area->storage_domain)  in iopt_area_unfill_domains()
1692 				area, domain, iopt_area_index(area),  in iopt_area_unfill_domains()
1693 				iopt_area_last_index(area));  in iopt_area_unfill_domains()
1696 		WARN_ON(RB_EMPTY_NODE(&area->pages_node.rb));  in iopt_area_unfill_domains()
1697 	interval_tree_remove(&area->pages_node, &pages->domains_itree);  in iopt_area_unfill_domains()
1698 	iopt_area_unfill_domain(area, pages, area->storage_domain);  in iopt_area_unfill_domains()
1699 	area->storage_domain = NULL;  in iopt_area_unfill_domains()
1802 		struct iopt_area *area;  in iopt_pages_fill_from_domain()  local
1804 		area = iopt_pages_find_domain_area(pages, start_index);  in iopt_pages_fill_from_domain()
1805 		if (WARN_ON(!area))  in iopt_pages_fill_from_domain()
1808 		domain_last = min(iopt_area_last_index(area), last_index);  in iopt_pages_fill_from_domain()
1809 		out_pages = raw_pages_from_domain(area->storage_domain, area,  in iopt_pages_fill_from_domain()
2102  * @area: The source of PFNs
2113 int iopt_area_add_access(struct iopt_area *area, unsigned long start_index,  in iopt_area_add_access()  argument
2117 	struct iopt_pages *pages = area->pages;  in iopt_area_add_access()
2127 		area->num_accesses++;  in iopt_area_add_access()
2148 	area->num_accesses++;  in iopt_area_add_access()
2162  * @area: The source of PFNs
2169 void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index,  in iopt_area_remove_access()  argument
2172 	struct iopt_pages *pages = area->pages;  in iopt_area_remove_access()
2180 	WARN_ON(area->num_accesses == 0 || access->users == 0);  in iopt_area_remove_access()
2181 	area->num_accesses--;  in iopt_area_remove_access()