332 	struct vm_struct *area;  in ioremap_page_range()  local
334 	area = find_vm_area((void *)addr);  in ioremap_page_range()
335 	if (!area || !(area->flags & VM_IOREMAP)) {  in ioremap_page_range()
339 	if (addr != (unsigned long)area->addr ||  in ioremap_page_range()
340 	    (void *)end != area->addr + get_vm_area_size(area)) {  in ioremap_page_range()
342 			  addr, end, (long)area->addr,  in ioremap_page_range()
343 			  (long)area->addr + get_vm_area_size(area));  in ioremap_page_range()
472  * @addr: start of the VM area to unmap
473  * @end: end of the VM area to unmap (non-inclusive)
646  * @addr: start of the VM area to map
647  * @end: end of the VM area to map (non-inclusive)
666 static int check_sparse_vm_area(struct vm_struct *area, unsigned long start,  in check_sparse_vm_area()  argument
670 	if (WARN_ON_ONCE(area->flags & VM_FLUSH_RESET_PERMS))  in check_sparse_vm_area()
672 	if (WARN_ON_ONCE(area->flags & VM_NO_GUARD))  in check_sparse_vm_area()
674 	if (WARN_ON_ONCE(!(area->flags & VM_SPARSE)))  in check_sparse_vm_area()
678 	if (start < (unsigned long)area->addr ||  in check_sparse_vm_area()
679 	    (void *)end > area->addr + get_vm_area_size(area))  in check_sparse_vm_area()
686  * @area: vm_area
691 int vm_area_map_pages(struct vm_struct *area, unsigned long start,  in vm_area_map_pages()  argument
696 	err = check_sparse_vm_area(area, start, end);  in vm_area_map_pages()
705  * @area: vm_area
709 void vm_area_unmap_pages(struct vm_struct *area, unsigned long start,  in vm_area_unmap_pages()  argument
712 	if (check_sparse_vm_area(area, start, end))  in vm_area_unmap_pages()
836  * find a lowest match of free area.
1355  * free area is inserted. If VA has been merged, it is
1403 			/* Point to the new merged area. */  in __merge_or_add_vmap_area()
1434 			/* Point to the new merged area. */  in __merge_or_add_vmap_area()
1755  * Returns a start address of the newly allocated area, if success.
1823 	 * when fit type of free area is NE_FIT_TYPE. It guarantees that  in preload_this_cpu_lock()
2343  * Free a vmap area, caller ensuring that the area has been unmapped,
2380  * Free and unmap a vmap area
2491 #define VMAP_RAM		0x1 /* indicates vm_map_ram area*/
3041  * vm_area_add_early - add vmap area early during boot
3044  * This function is used to add fixed kernel vm area to vmlist before
3067  * vm_area_register_early - register vmap area early during boot
3071  * This function is used to register kernel vm area before
3115 	struct vm_struct *area;  in __get_vm_area_node()  local
3127 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);  in __get_vm_area_node()
3128 	if (unlikely(!area))  in __get_vm_area_node()
3134 	area->flags = flags;  in __get_vm_area_node()
3135 	area->caller = caller;  in __get_vm_area_node()
3137 	va = alloc_vmap_area(size, align, start, end, node, gfp_mask, 0, area);  in __get_vm_area_node()
3139 		kfree(area);  in __get_vm_area_node()
3152 		area->addr = kasan_unpoison_vmalloc(area->addr, requested_size,  in __get_vm_area_node()
3155 	return area;  in __get_vm_area_node()
3167  * get_vm_area - reserve a contiguous kernel virtual area
3168  * @size:	 size of the area
3171  * Search an area of @size in the kernel virtual mapping area,
3172  * and reserved it for out purposes.  Returns the area descriptor
3175  * Return: the area descriptor on success or %NULL on failure.
3194  * find_vm_area - find a continuous kernel virtual area
3197  * Search for the kernel VM area starting at @addr, and return it.
3201  * Return: the area descriptor on success or %NULL on failure.
3215  * remove_vm_area - find and remove a continuous kernel virtual area
3218  * Search for the kernel VM area starting at @addr, and remove it.
3219  * This function returns the found VM area, but using it is NOT safe
3222  * Return: the area descriptor on success or %NULL on failure.
3249 static inline void set_area_direct_map(const struct vm_struct *area,  in set_area_direct_map()  argument
3255 	for (i = 0; i < area->nr_pages; i++)  in set_area_direct_map()
3256 		if (page_address(area->pages[i]))  in set_area_direct_map()
3257 			set_direct_map(area->pages[i]);  in set_area_direct_map()
3263 static void vm_reset_perms(struct vm_struct *area)  in vm_reset_perms()  argument
3266 	unsigned int page_order = vm_area_page_order(area);  in vm_reset_perms()
3274 	for (i = 0; i < area->nr_pages; i += 1U << page_order) {  in vm_reset_perms()
3275 		unsigned long addr = (unsigned long)page_address(area->pages[i]);  in vm_reset_perms()
3292 	set_area_direct_map(area, set_direct_map_invalid_noflush);  in vm_reset_perms()
3294 	set_area_direct_map(area, set_direct_map_default_noflush);  in vm_reset_perms()
3334  * Free the virtually continuous memory area starting at @addr, as obtained
3366 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",  in vfree()
3397  * Free the virtually contiguous memory area starting at @addr,
3413 		WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n",  in vunmap()
3434  * Return: the address of the area or %NULL on failure
3439 	struct vm_struct *area;  in vmap()  local
3459 	area = get_vm_area_caller(size, flags, __builtin_return_address(0));  in vmap()
3460 	if (!area)  in vmap()
3463 	addr = (unsigned long)area->addr;  in vmap()
3466 		vunmap(area->addr);  in vmap()
3471 		area->pages = pages;  in vmap()
3472 		area->nr_pages = count;  in vmap()
3474 	return area->addr;  in vmap()
3513 	struct vm_struct *area;  in vmap_pfn()  local
3515 	area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP,  in vmap_pfn()
3517 	if (!area)  in vmap_pfn()
3519 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,  in vmap_pfn()
3521 		free_vm_area(area);  in vmap_pfn()
3525 	flush_cache_vmap((unsigned long)area->addr,  in vmap_pfn()
3526 			 (unsigned long)area->addr + count * PAGE_SIZE);  in vmap_pfn()
3528 	return area->addr;  in vmap_pfn()
3623 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,  in __vmalloc_area_node()  argument
3629 	unsigned long addr = (unsigned long)area->addr;  in __vmalloc_area_node()
3630 	unsigned long size = get_vm_area_size(area);  in __vmalloc_area_node()
3644 		area->pages = __vmalloc_node_noprof(array_size, 1, nested_gfp, node,  in __vmalloc_area_node()
3645 					area->caller);  in __vmalloc_area_node()
3647 		area->pages = kmalloc_node_noprof(array_size, nested_gfp, node);  in __vmalloc_area_node()
3650 	if (!area->pages) {  in __vmalloc_area_node()
3654 		free_vm_area(area);  in __vmalloc_area_node()
3658 	set_vm_area_page_order(area, page_shift - PAGE_SHIFT);  in __vmalloc_area_node()
3659 	page_order = vm_area_page_order(area);  in __vmalloc_area_node()
3669 	area->nr_pages = vm_area_alloc_pages((page_order ?  in __vmalloc_area_node()
3671 		node, page_order, nr_small_pages, area->pages);  in __vmalloc_area_node()
3673 	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
3677 		for (i = 0; i < area->nr_pages; i++)  in __vmalloc_area_node()
3678 			mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1);  in __vmalloc_area_node()
3685 	if (area->nr_pages != nr_small_pages) {  in __vmalloc_area_node()
3699 				area->nr_pages * PAGE_SIZE);  in __vmalloc_area_node()
3713 		ret = vmap_pages_range(addr, addr + size, prot, area->pages,  in __vmalloc_area_node()
3727 			area->nr_pages * PAGE_SIZE);  in __vmalloc_area_node()
3731 	return area->addr;  in __vmalloc_area_node()
3734 	vfree(area->addr);  in __vmalloc_area_node()
3742  * @start:		  vm area range start
3743  * @end:		  vm area range end
3746  * @vm_flags:		  additional vm area flags (e.g. %VM_NO_GUARD)
3764  * Return: the address of the area or %NULL on failure
3771 	struct vm_struct *area;  in __vmalloc_node_range_noprof()  local
3806 	area = __get_vm_area_node(real_size, align, shift, VM_ALLOC |  in __vmalloc_node_range_noprof()
3809 	if (!area) {  in __vmalloc_node_range_noprof()
3846 	ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node);  in __vmalloc_node_range_noprof()
3863 	area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags);  in __vmalloc_node_range_noprof()
3870 	clear_vm_uninitialized_flag(area);  in __vmalloc_node_range_noprof()
3874 		kmemleak_vmalloc(area, size, gfp_mask);  in __vmalloc_node_range_noprof()
3876 	return area->addr;  in __vmalloc_node_range_noprof()
3993  * The resulting memory area is zeroed so it can be mapped to userspace
4083 			WARN(1, "Trying to vrealloc() nonexistent vm area (%p)\n", p);  in vrealloc_noprof()
4148  * The resulting memory area is 32bit addressable and zeroed so it can be
4207 		 * To do safe access to this _mapped_ area, we need lock. But  in aligned_vread_iter()
4246 	 * If it's area created by vm_map_ram() interface directly, but  in vmap_ram_vread_iter()
4256 	 * Area is split into regions and tracked with vmap_block, read out  in vmap_ram_vread_iter()
4316  * vread_iter() - read vmalloc area in a safe way to an iterator.
4321  * This function checks that addr is a valid vmalloc'ed area, and
4322  * copy data from that area to a given buffer. If the given memory range
4324  * proper area of @buf. If there are memory holes, they'll be zero-filled.
4325  * IOREMAP area is treated as memory hole and no copy is done.
4328  * vm_struct area, returns 0. @buf should be kernel's buffer.
4331  * should know vmalloc() area is valid and can use memcpy().
4332  * This is for routines which have to access vmalloc area without
4337  * include any intersection with valid vmalloc area
4373 		 * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need  in vread_iter()
4412 		else /* IOREMAP | SPARSE area is treated as memory hole */  in vread_iter()
4446  * @size:		size of map area
4450  * This function checks that @kaddr is a valid vmalloc'ed area,
4461 	struct vm_struct *area;  in remap_vmalloc_range_partial()  local
4473 	area = find_vm_area(kaddr);  in remap_vmalloc_range_partial()
4474 	if (!area)  in remap_vmalloc_range_partial()
4477 	if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))  in remap_vmalloc_range_partial()
4481 	    end_index > get_vm_area_size(area))  in remap_vmalloc_range_partial()
4511  * This function checks that addr is a valid vmalloc'ed area, and
4526 void free_vm_area(struct vm_struct *area)  in free_vm_area()  argument
4529 	ret = remove_vm_area(area->addr);  in free_vm_area()
4530 	BUG_ON(ret != area);  in free_vm_area()
4531 	kfree(area);  in free_vm_area()
4545  * Returns: vmap_area if it is found. If there is no such area
4605  * @offsets: array containing offset of each area
4606  * @sizes: array containing size of each area
4623  * base address is pulled down to fit the area. Scanning is repeated till
4635 	int area, area2, last_area, term_area;  in pcpu_get_vm_areas()  local
4641 	for (last_area = 0, area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4642 		start = offsets[area];  in pcpu_get_vm_areas()
4643 		end = start + sizes[area];  in pcpu_get_vm_areas()
4646 		BUG_ON(!IS_ALIGNED(offsets[area], align));  in pcpu_get_vm_areas()
4647 		BUG_ON(!IS_ALIGNED(sizes[area], align));  in pcpu_get_vm_areas()
4649 		/* detect the area with the highest address */  in pcpu_get_vm_areas()
4651 			last_area = area;  in pcpu_get_vm_areas()
4653 		for (area2 = area + 1; area2 < nr_vms; area2++) {  in pcpu_get_vm_areas()
4672 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4673 		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);  in pcpu_get_vm_areas()
4674 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);  in pcpu_get_vm_areas()
4675 		if (!vas[area] || !vms[area])  in pcpu_get_vm_areas()
4681 	/* start scanning - we scan from the top, begin with the last area */  in pcpu_get_vm_areas()
4682 	area = term_area = last_area;  in pcpu_get_vm_areas()
4683 	start = offsets[area];  in pcpu_get_vm_areas()
4684 	end = start + sizes[area];  in pcpu_get_vm_areas()
4709 			term_area = area;  in pcpu_get_vm_areas()
4719 			term_area = area;  in pcpu_get_vm_areas()
4724 		 * This area fits, move on to the previous one.  If  in pcpu_get_vm_areas()
4727 		area = (area + nr_vms - 1) % nr_vms;  in pcpu_get_vm_areas()
4728 		if (area == term_area)  in pcpu_get_vm_areas()
4731 		start = offsets[area];  in pcpu_get_vm_areas()
4732 		end = start + sizes[area];  in pcpu_get_vm_areas()
4737 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4740 		start = base + offsets[area];  in pcpu_get_vm_areas()
4741 		size = sizes[area];  in pcpu_get_vm_areas()
4754 		/* Allocated area. */  in pcpu_get_vm_areas()
4755 		va = vas[area];  in pcpu_get_vm_areas()
4763 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4764 		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))  in pcpu_get_vm_areas()
4769 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4770 		struct vmap_node *vn = addr_to_node(vas[area]->va_start);  in pcpu_get_vm_areas()
4773 		insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head);  in pcpu_get_vm_areas()
4774 		setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,  in pcpu_get_vm_areas()
4785 	for (area = 0; area < nr_vms; area++)  in pcpu_get_vm_areas()
4786 		vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr,  in pcpu_get_vm_areas()
4787 				vms[area]->size, KASAN_VMALLOC_PROT_NORMAL);  in pcpu_get_vm_areas()
4799 	while (area--) {  in pcpu_get_vm_areas()
4800 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
4801 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
4802 		va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
4808 		vas[area] = NULL;  in pcpu_get_vm_areas()
4818 		for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4819 			if (vas[area])  in pcpu_get_vm_areas()
4822 			vas[area] = kmem_cache_zalloc(  in pcpu_get_vm_areas()
4824 			if (!vas[area])  in pcpu_get_vm_areas()
4832 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4833 		if (vas[area])  in pcpu_get_vm_areas()
4834 			kmem_cache_free(vmap_area_cachep, vas[area]);  in pcpu_get_vm_areas()
4836 		kfree(vms[area]);  in pcpu_get_vm_areas()
4850 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4851 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
4852 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
4853 		va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
4859 		vas[area] = NULL;  in pcpu_get_vm_areas()
4860 		kfree(vms[area]);  in pcpu_get_vm_areas()