130 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)  in io_netmsg_iovec_free()  argument
132 	if (kmsg->free_iov) {  in io_netmsg_iovec_free()
133 		kfree(kmsg->free_iov);  in io_netmsg_iovec_free()
134 		kmsg->free_iov_nr = 0;  in io_netmsg_iovec_free()
135 		kmsg->free_iov = NULL;  in io_netmsg_iovec_free()
172 /* assign new iovec to kmsg, if we need to */
173 static void io_net_vec_assign(struct io_kiocb *req, struct io_async_msghdr *kmsg,  in io_net_vec_assign()  argument
178 		kmsg->free_iov_nr = kmsg->msg.msg_iter.nr_segs;  in io_net_vec_assign()
179 		if (kmsg->free_iov)  in io_net_vec_assign()
180 			kfree(kmsg->free_iov);  in io_net_vec_assign()
181 		kmsg->free_iov = iov;  in io_net_vec_assign()
186 				       struct io_async_msghdr *kmsg)  in io_mshot_prep_retry()  argument
355 	struct io_async_msghdr *kmsg = req->async_data;  in io_send_setup()  local
365 	kmsg->msg.msg_name = NULL;  in io_send_setup()
366 	kmsg->msg.msg_namelen = 0;  in io_send_setup()
367 	kmsg->msg.msg_control = NULL;  in io_send_setup()
368 	kmsg->msg.msg_controllen = 0;  in io_send_setup()
369 	kmsg->msg.msg_ubuf = NULL;  in io_send_setup()
374 		ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);  in io_send_setup()
377 		kmsg->msg.msg_name = &kmsg->addr;  in io_send_setup()
378 		kmsg->msg.msg_namelen = addr_len;  in io_send_setup()
382 				  &kmsg->msg.msg_iter);  in io_send_setup()
392 	struct io_async_msghdr *kmsg = req->async_data;  in io_sendmsg_setup()  local
397 	ret = io_sendmsg_copy_hdr(req, kmsg);  in io_sendmsg_setup()
460 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret)  in io_bundle_nbufs()  argument
468 	if (iter_is_ubuf(&kmsg->msg.msg_iter))  in io_bundle_nbufs()
471 	iov = kmsg->free_iov;  in io_bundle_nbufs()
473 		iov = &kmsg->fast_iov;  in io_bundle_nbufs()
476 	if (!iov_iter_count(&kmsg->msg.msg_iter))  in io_bundle_nbufs()
477 		return iter_iov(&kmsg->msg.msg_iter) - iov;  in io_bundle_nbufs()
492 				  struct io_async_msghdr *kmsg,  in io_send_finish()  argument
504 	cflags = io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret), issue_flags);  in io_send_finish()
514 		io_mshot_prep_retry(req, kmsg);  in io_send_finish()
528 	struct io_async_msghdr *kmsg = req->async_data;  in io_sendmsg()  local
546 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_sendmsg()
548 	kmsg->msg.msg_control_user = sr->msg_control;  in io_sendmsg()
550 	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);  in io_sendmsg()
556 			kmsg->msg.msg_controllen = 0;  in io_sendmsg()
557 			kmsg->msg.msg_control = NULL;  in io_sendmsg()
576 				 struct io_async_msghdr *kmsg)  in io_send_select_buffer()  argument
582 		.iovs = &kmsg->fast_iov,  in io_send_select_buffer()
587 	if (kmsg->free_iov) {  in io_send_select_buffer()
588 		arg.nr_iovs = kmsg->free_iov_nr;  in io_send_select_buffer()
589 		arg.iovs = kmsg->free_iov;  in io_send_select_buffer()
602 	if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) {  in io_send_select_buffer()
603 		kmsg->free_iov_nr = ret;  in io_send_select_buffer()
604 		kmsg->free_iov = arg.iovs;  in io_send_select_buffer()
612 					&kmsg->msg.msg_iter);  in io_send_select_buffer()
616 		iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,  in io_send_select_buffer()
626 	struct io_async_msghdr *kmsg = req->async_data;  in io_send()  local
646 		ret = io_send_select_buffer(req, issue_flags, kmsg);  in io_send()
657 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_send()
660 	kmsg->msg.msg_flags = flags;  in io_send()
661 	ret = sock_sendmsg(sock, &kmsg->msg);  in io_send()
682 	if (!io_send_finish(req, &ret, kmsg, issue_flags))  in io_send()
754 	struct io_async_msghdr *kmsg;  in io_recvmsg_prep_setup()  local
757 	kmsg = io_msg_alloc_async(req);  in io_recvmsg_prep_setup()
758 	if (unlikely(!kmsg))  in io_recvmsg_prep_setup()
762 		kmsg->msg.msg_name = NULL;  in io_recvmsg_prep_setup()
763 		kmsg->msg.msg_namelen = 0;  in io_recvmsg_prep_setup()
764 		kmsg->msg.msg_inq = 0;  in io_recvmsg_prep_setup()
765 		kmsg->msg.msg_control = NULL;  in io_recvmsg_prep_setup()
766 		kmsg->msg.msg_get_inq = 1;  in io_recvmsg_prep_setup()
767 		kmsg->msg.msg_controllen = 0;  in io_recvmsg_prep_setup()
768 		kmsg->msg.msg_iocb = NULL;  in io_recvmsg_prep_setup()
769 		kmsg->msg.msg_ubuf = NULL;  in io_recvmsg_prep_setup()
773 					  &kmsg->msg.msg_iter);  in io_recvmsg_prep_setup()
780 	ret = io_recvmsg_copy_hdr(req, kmsg);  in io_recvmsg_prep_setup()
853 				  struct io_async_msghdr *kmsg,  in io_recv_finish()  argument
859 	if (kmsg->msg.msg_inq > 0)  in io_recv_finish()
863 		cflags |= io_put_kbufs(req, *ret, io_bundle_nbufs(kmsg, *ret),  in io_recv_finish()
871 		if (!sr->retry && kmsg->msg.msg_inq > 0 && *ret > 0) {  in io_recv_finish()
873 			sr->len = kmsg->msg.msg_inq;  in io_recv_finish()
890 		io_mshot_prep_retry(req, kmsg);  in io_recv_finish()
892 		if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {  in io_recv_finish()
918 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg,  in io_recvmsg_prep_multishot()  argument
925 	hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +  in io_recvmsg_prep_multishot()
926 		kmsg->controllen;  in io_recvmsg_prep_multishot()
930 	if (kmsg->controllen) {  in io_recvmsg_prep_multishot()
931 		unsigned long control = ubuf + hdr - kmsg->controllen;  in io_recvmsg_prep_multishot()
933 		kmsg->msg.msg_control_user = (void __user *) control;  in io_recvmsg_prep_multishot()
934 		kmsg->msg.msg_controllen = kmsg->controllen;  in io_recvmsg_prep_multishot()
939 	kmsg->payloadlen = *len = *len - hdr;  in io_recvmsg_prep_multishot()
949 				struct io_async_msghdr *kmsg,  in io_recvmsg_multishot()  argument
956 	if (kmsg->namelen)  in io_recvmsg_multishot()
957 		kmsg->msg.msg_name = &hdr.addr;  in io_recvmsg_multishot()
958 	kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);  in io_recvmsg_multishot()
959 	kmsg->msg.msg_namelen = 0;  in io_recvmsg_multishot()
964 	err = sock_recvmsg(sock, &kmsg->msg, flags);  in io_recvmsg_multishot()
970 		.controllen = kmsg->controllen - kmsg->msg.msg_controllen,  in io_recvmsg_multishot()
971 		.flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT  in io_recvmsg_multishot()
975 	if (err > kmsg->payloadlen)  in io_recvmsg_multishot()
976 		err = kmsg->payloadlen;  in io_recvmsg_multishot()
979 	if (kmsg->msg.msg_namelen > kmsg->namelen)  in io_recvmsg_multishot()
980 		copy_len += kmsg->namelen;  in io_recvmsg_multishot()
982 		copy_len += kmsg->msg.msg_namelen;  in io_recvmsg_multishot()
988 	hdr.msg.namelen = kmsg->msg.msg_namelen;  in io_recvmsg_multishot()
998 	return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +  in io_recvmsg_multishot()
999 			kmsg->controllen + err;  in io_recvmsg_multishot()
1005 	struct io_async_msghdr *kmsg = req->async_data;  in io_recvmsg()  local
1034 			ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);  in io_recvmsg()
1041 		iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);  in io_recvmsg()
1044 	kmsg->msg.msg_get_inq = 1;  in io_recvmsg()
1045 	kmsg->msg.msg_inq = -1;  in io_recvmsg()
1047 		ret = io_recvmsg_multishot(sock, sr, kmsg, flags,  in io_recvmsg()
1051 		if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)  in io_recvmsg()
1052 			min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_recvmsg()
1054 		ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,  in io_recvmsg()
1055 					 kmsg->uaddr, flags);  in io_recvmsg()
1074 	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {  in io_recvmsg()
1085 	if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))  in io_recvmsg()
1091 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg,  in io_recv_buf_select()  argument
1105 			.iovs = &kmsg->fast_iov,  in io_recv_buf_select()
1110 		if (kmsg->free_iov) {  in io_recv_buf_select()
1111 			arg.nr_iovs = kmsg->free_iov_nr;  in io_recv_buf_select()
1112 			arg.iovs = kmsg->free_iov;  in io_recv_buf_select()
1116 		if (kmsg->msg.msg_inq > 0)  in io_recv_buf_select()
1117 			arg.max_len = min_not_zero(sr->len, kmsg->msg.msg_inq);  in io_recv_buf_select()
1129 		iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,  in io_recv_buf_select()
1131 		if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->free_iov) {  in io_recv_buf_select()
1132 			kmsg->free_iov_nr = ret;  in io_recv_buf_select()
1133 			kmsg->free_iov = arg.iovs;  in io_recv_buf_select()
1147 				  &kmsg->msg.msg_iter);  in io_recv_buf_select()
1158 	struct io_async_msghdr *kmsg = req->async_data;  in io_recv()  local
1180 		ret = io_recv_buf_select(req, kmsg, &len, issue_flags);  in io_recv()
1182 			kmsg->msg.msg_inq = -1;  in io_recv()
1188 	kmsg->msg.msg_flags = 0;  in io_recv()
1189 	kmsg->msg.msg_inq = -1;  in io_recv()
1192 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_recv()
1194 	ret = sock_recvmsg(sock, &kmsg->msg, flags);  in io_recv()
1214 	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {  in io_recv()
1227 	if (!io_recv_finish(req, &ret, kmsg, mshot_finished, issue_flags))  in io_recv()
1364 	struct io_async_msghdr *kmsg = req->async_data;  in io_send_zc_import()  local
1383 		ret = io_import_fixed(ITER_SOURCE, &kmsg->msg.msg_iter,  in io_send_zc_import()
1388 		kmsg->msg.sg_from_iter = io_sg_from_iter;  in io_send_zc_import()
1390 		ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);  in io_send_zc_import()
1396 		kmsg->msg.sg_from_iter = io_sg_from_iter_iovec;  in io_send_zc_import()
1405 	struct io_async_msghdr *kmsg = req->async_data;  in io_send_zc()  local
1431 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_send_zc()
1434 	kmsg->msg.msg_flags = msg_flags;  in io_send_zc()
1435 	kmsg->msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;  in io_send_zc()
1436 	ret = sock_sendmsg(sock, &kmsg->msg);  in io_send_zc()
1442 		if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {  in io_send_zc()
1475 	struct io_async_msghdr *kmsg = req->async_data;  in io_sendmsg_zc()  local
1494 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);  in io_sendmsg_zc()
1496 	kmsg->msg.msg_control_user = sr->msg_control;  in io_sendmsg_zc()
1497 	kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;  in io_sendmsg_zc()
1498 	kmsg->msg.sg_from_iter = io_sg_from_iter_iovec;  in io_sendmsg_zc()
1499 	ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);  in io_sendmsg_zc()
1846 	struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry;  in io_netmsg_cache_free()  local
1848 	if (kmsg->free_iov)  in io_netmsg_cache_free()
1849 		io_netmsg_iovec_free(kmsg);  in io_netmsg_cache_free()
1850 	kfree(kmsg);  in io_netmsg_cache_free()