124 int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len,
128 void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len);
129 int sk_msg_free(struct sock *sk, struct sk_msg *msg);
130 int sk_msg_free_nocharge(struct sock *sk, struct sk_msg *msg);
131 void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes);
132 void sk_msg_free_partial_nocharge(struct sock *sk, struct sk_msg *msg,
135 void sk_msg_return(struct sock *sk, struct sk_msg *msg, int bytes);
136 void sk_msg_return_zero(struct sock *sk, struct sk_msg *msg, int bytes);
139 			      struct sk_msg *msg, u32 bytes);
141 			     struct sk_msg *msg, u32 bytes);
142 int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
146 static inline void sk_msg_check_to_free(struct sk_msg *msg, u32 i, u32 bytes)  in sk_msg_check_to_free()  argument
148 	WARN_ON(i == msg->sg.end && bytes);  in sk_msg_check_to_free()
181 #define sk_msg_iter_prev(msg, which)			\  argument
182 	sk_msg_iter_var_prev(msg->sg.which)
184 #define sk_msg_iter_next(msg, which)			\  argument
185 	sk_msg_iter_var_next(msg->sg.which)
187 static inline void sk_msg_init(struct sk_msg *msg)  in sk_msg_init()  argument
189 	BUILD_BUG_ON(ARRAY_SIZE(msg->sg.data) - 1 != NR_MSG_FRAG_IDS);  in sk_msg_init()
190 	memset(msg, 0, sizeof(*msg));  in sk_msg_init()
191 	sg_init_marker(msg->sg.data, NR_MSG_FRAG_IDS);  in sk_msg_init()
211 static inline bool sk_msg_full(const struct sk_msg *msg)  in sk_msg_full()  argument
213 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end) == MAX_MSG_FRAGS;  in sk_msg_full()
216 static inline u32 sk_msg_elem_used(const struct sk_msg *msg)  in sk_msg_elem_used()  argument
218 	return sk_msg_iter_dist(msg->sg.start, msg->sg.end);  in sk_msg_elem_used()
221 static inline struct scatterlist *sk_msg_elem(struct sk_msg *msg, int which)  in sk_msg_elem()  argument
223 	return &msg->sg.data[which];  in sk_msg_elem()
226 static inline struct scatterlist sk_msg_elem_cpy(struct sk_msg *msg, int which)  in sk_msg_elem_cpy()  argument
228 	return msg->sg.data[which];  in sk_msg_elem_cpy()
231 static inline struct page *sk_msg_page(struct sk_msg *msg, int which)  in sk_msg_page()  argument
233 	return sg_page(sk_msg_elem(msg, which));  in sk_msg_page()
236 static inline bool sk_msg_to_ingress(const struct sk_msg *msg)  in sk_msg_to_ingress()  argument
238 	return msg->flags & BPF_F_INGRESS;  in sk_msg_to_ingress()
241 static inline void sk_msg_compute_data_pointers(struct sk_msg *msg)  in sk_msg_compute_data_pointers()  argument
243 	struct scatterlist *sge = sk_msg_elem(msg, msg->sg.start);  in sk_msg_compute_data_pointers()
245 	if (test_bit(msg->sg.start, msg->sg.copy)) {  in sk_msg_compute_data_pointers()
246 		msg->data = NULL;  in sk_msg_compute_data_pointers()
247 		msg->data_end = NULL;  in sk_msg_compute_data_pointers()
249 		msg->data = sg_virt(sge);  in sk_msg_compute_data_pointers()
250 		msg->data_end = msg->data + sge->length;  in sk_msg_compute_data_pointers()
254 static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page,  in sk_msg_page_add()  argument
260 	sge = sk_msg_elem(msg, msg->sg.end);  in sk_msg_page_add()
264 	__set_bit(msg->sg.end, msg->sg.copy);  in sk_msg_page_add()
265 	msg->sg.size += len;  in sk_msg_page_add()
266 	sk_msg_iter_next(msg, end);  in sk_msg_page_add()
269 static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state)  in sk_msg_sg_copy()  argument
273 			__set_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
275 			__clear_bit(i, msg->sg.copy);  in sk_msg_sg_copy()
277 		if (i == msg->sg.end)  in sk_msg_sg_copy()
282 static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_set()  argument
284 	sk_msg_sg_copy(msg, start, true);  in sk_msg_sg_copy_set()
287 static inline void sk_msg_sg_copy_clear(struct sk_msg *msg, u32 start)  in sk_msg_sg_copy_clear()  argument
289 	sk_msg_sg_copy(msg, start, false);  in sk_msg_sg_copy_clear()
323 				      struct sk_msg *msg)  in sk_psock_queue_msg()  argument
329 		list_add_tail(&msg->list, &psock->ingress_msg);  in sk_psock_queue_msg()
332 		sk_msg_free(psock->sk, msg);  in sk_psock_queue_msg()
333 		kfree(msg);  in sk_psock_queue_msg()
342 	struct sk_msg *msg;  in sk_psock_dequeue_msg()  local
345 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_dequeue_msg()
346 	if (msg)  in sk_psock_dequeue_msg()
347 		list_del(&msg->list);  in sk_psock_dequeue_msg()
349 	return msg;  in sk_psock_dequeue_msg()
354 	struct sk_msg *msg;  in sk_psock_peek_msg()  local
357 	msg = list_first_entry_or_null(&psock->ingress_msg, struct sk_msg, list);  in sk_psock_peek_msg()
359 	return msg;  in sk_psock_peek_msg()
363 					       struct sk_msg *msg)  in sk_psock_next_msg()  argument
368 	if (list_is_last(&msg->list, &psock->ingress_msg))  in sk_psock_next_msg()
371 		ret = list_next_entry(msg, list);  in sk_psock_next_msg()
381 static inline void kfree_sk_msg(struct sk_msg *msg)  in kfree_sk_msg()  argument
383 	if (msg->skb)  in kfree_sk_msg()
384 		consume_skb(msg->skb);  in kfree_sk_msg()
385 	kfree(msg);  in kfree_sk_msg()
422 			 struct sk_msg *msg);