765 static void pfn_reader_user_init(struct pfn_reader_user *user,  in pfn_reader_user_init()  argument
768 	user->upages = NULL;  in pfn_reader_user_init()
769 	user->upages_len = 0;  in pfn_reader_user_init()
770 	user->upages_start = 0;  in pfn_reader_user_init()
771 	user->upages_end = 0;  in pfn_reader_user_init()
772 	user->locked = -1;  in pfn_reader_user_init()
773 	user->gup_flags = FOLL_LONGTERM;  in pfn_reader_user_init()
775 		user->gup_flags |= FOLL_WRITE;  in pfn_reader_user_init()
777 	user->file = (pages->type == IOPT_ADDRESS_FILE) ? pages->file : NULL;  in pfn_reader_user_init()
778 	user->ufolios = NULL;  in pfn_reader_user_init()
779 	user->ufolios_len = 0;  in pfn_reader_user_init()
780 	user->ufolios_next = NULL;  in pfn_reader_user_init()
781 	user->ufolios_offset = 0;  in pfn_reader_user_init()
784 static void pfn_reader_user_destroy(struct pfn_reader_user *user,  in pfn_reader_user_destroy()  argument
787 	if (user->locked != -1) {  in pfn_reader_user_destroy()
788 		if (user->locked)  in pfn_reader_user_destroy()
790 		if (!user->file && pages->source_mm != current->mm)  in pfn_reader_user_destroy()
792 		user->locked = -1;  in pfn_reader_user_destroy()
795 	kfree(user->upages);  in pfn_reader_user_destroy()
796 	user->upages = NULL;  in pfn_reader_user_destroy()
797 	kfree(user->ufolios);  in pfn_reader_user_destroy()
798 	user->ufolios = NULL;  in pfn_reader_user_destroy()
801 static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start,  in pin_memfd_pages()  argument
807 	struct page **upages = user->upages;  in pin_memfd_pages()
809 	long nfolios = user->ufolios_len / sizeof(*user->ufolios);  in pin_memfd_pages()
816 	nfolios = memfd_pin_folios(user->file, start, end, user->ufolios,  in pin_memfd_pages()
822 	user->ufolios_next = user->ufolios;  in pin_memfd_pages()
823 	user->ufolios_offset = offset;  in pin_memfd_pages()
826 		struct folio *folio = user->ufolios[i];  in pin_memfd_pages()
853 static int pfn_reader_user_pin(struct pfn_reader_user *user,  in pfn_reader_user_pin()  argument
869 	if (!user->file && !user->upages) {  in pfn_reader_user_pin()
871 		user->upages_len = npages * sizeof(*user->upages);  in pfn_reader_user_pin()
872 		user->upages = temp_kmalloc(&user->upages_len, NULL, 0);  in pfn_reader_user_pin()
873 		if (!user->upages)  in pfn_reader_user_pin()
877 	if (user->file && !user->ufolios) {  in pfn_reader_user_pin()
878 		user->ufolios_len = npages * sizeof(*user->ufolios);  in pfn_reader_user_pin()
879 		user->ufolios = temp_kmalloc(&user->ufolios_len, NULL, 0);  in pfn_reader_user_pin()
880 		if (!user->ufolios)  in pfn_reader_user_pin()
884 	if (user->locked == -1) {  in pfn_reader_user_pin()
890 		if (!user->file && remote_mm) {  in pfn_reader_user_pin()
894 		user->locked = 0;  in pfn_reader_user_pin()
897 	unum = user->file ? user->ufolios_len / sizeof(*user->ufolios) :  in pfn_reader_user_pin()
898 			    user->upages_len / sizeof(*user->upages);  in pfn_reader_user_pin()
904 	if (user->file) {  in pfn_reader_user_pin()
906 		rc = pin_memfd_pages(user, start, npages);  in pfn_reader_user_pin()
909 		rc = pin_user_pages_fast(uptr, npages, user->gup_flags,  in pfn_reader_user_pin()
910 					 user->upages);  in pfn_reader_user_pin()
913 		if (!user->locked) {  in pfn_reader_user_pin()
915 			user->locked = 1;  in pfn_reader_user_pin()
918 					   user->gup_flags, user->upages,  in pfn_reader_user_pin()
919 					   &user->locked);  in pfn_reader_user_pin()
927 	user->upages_start = start_index;  in pfn_reader_user_pin()
928 	user->upages_end = start_index + rc;  in pfn_reader_user_pin()
961 			       bool inc, struct pfn_reader_user *user)  in update_mm_locked_vm()  argument
966 	if (user && user->locked) {  in update_mm_locked_vm()
968 		user->locked = 0;  in update_mm_locked_vm()
971 	} else if ((!user || (!user->upages && !user->ufolios)) &&  in update_mm_locked_vm()
989 			     bool inc, struct pfn_reader_user *user)  in iopt_pages_update_pinned()  argument
1003 		rc = update_mm_locked_vm(pages, npages, inc, user);  in iopt_pages_update_pinned()
1029  * and processed. If the user lacked the limit then the error unwind will unpin
1034 static int pfn_reader_user_update_pinned(struct pfn_reader_user *user,  in pfn_reader_user_update_pinned()  argument
1054 	return iopt_pages_update_pinned(pages, npages, inc, user);  in pfn_reader_user_update_pinned()
1075 	struct pfn_reader_user user;  member
1080 	return pfn_reader_user_update_pinned(&pfns->user, pfns->pages);  in pfn_reader_update_pinned()
1111 	struct pfn_reader_user *user = &pfns->user;  in pfn_reader_fill_span()  local
1143 	if (start_index >= pfns->user.upages_end) {  in pfn_reader_fill_span()
1144 		rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index,  in pfn_reader_fill_span()
1150 	npages = user->upages_end - start_index;  in pfn_reader_fill_span()
1151 	start_index -= user->upages_start;  in pfn_reader_fill_span()
1154 	if (!user->file)  in pfn_reader_fill_span()
1155 		batch_from_pages(&pfns->batch, user->upages + start_index,  in pfn_reader_fill_span()
1158 		rc = batch_from_folios(&pfns->batch, &user->ufolios_next,  in pfn_reader_fill_span()
1159 				       &user->ufolios_offset, npages);  in pfn_reader_fill_span()
1212 	pfn_reader_user_init(&pfns->user, pages);  in pfn_reader_init()
1233 	struct pfn_reader_user *user = &pfns->user;  in pfn_reader_release_pins()  local
1235 	if (user->upages_end > pfns->batch_end_index) {  in pfn_reader_release_pins()
1238 		unsigned long npages = user->upages_end - pfns->batch_end_index;  in pfn_reader_release_pins()
1240 					    user->upages_start;  in pfn_reader_release_pins()
1242 		if (!user->file) {  in pfn_reader_release_pins()
1243 			unpin_user_pages(user->upages + start_index, npages);  in pfn_reader_release_pins()
1245 			long n = user->ufolios_len / sizeof(*user->ufolios);  in pfn_reader_release_pins()
1247 			unpin_folios(user->ufolios_next,  in pfn_reader_release_pins()
1248 				     user->ufolios + n - user->ufolios_next);  in pfn_reader_release_pins()
1251 		user->upages_end = pfns->batch_end_index;  in pfn_reader_release_pins()
1264 	pfn_reader_user_destroy(&pfns->user, pfns->pages);  in pfn_reader_destroy()
1818 			   struct pfn_reader_user *user,  in iopt_pages_fill()  argument
1827 		user->upages = out_pages + (cur_index - start_index);  in iopt_pages_fill()
1828 		rc = pfn_reader_user_pin(user, pages, cur_index, last_index);  in iopt_pages_fill()
1831 		cur_index = user->upages_end;  in iopt_pages_fill()
1861 	struct pfn_reader_user user;  in iopt_pages_fill_xarray()  local
1866 	pfn_reader_user_init(&user, pages);  in iopt_pages_fill_xarray()
1867 	user.upages_len = (last_index - start_index + 1) * sizeof(*out_pages);  in iopt_pages_fill_xarray()
1895 		rc = iopt_pages_fill(pages, &user, span.start_hole,  in iopt_pages_fill_xarray()
1908 	rc = pfn_reader_user_update_pinned(&user, pages);  in iopt_pages_fill_xarray()
1911 	user.upages = NULL;  in iopt_pages_fill_xarray()
1912 	pfn_reader_user_destroy(&user, pages);  in iopt_pages_fill_xarray()
1918 	user.upages = NULL;  in iopt_pages_fill_xarray()
1919 	pfn_reader_user_destroy(&user, pages);  in iopt_pages_fill_xarray()