6 #include <net/xdp.h>
176 static u16 igb_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,  in igb_fill_rx_descs()  argument
187 	buffs = xsk_buff_alloc_batch(pool, xdp, count);  in igb_fill_rx_descs()
189 		dma = xsk_buff_xdp_get_dma(*xdp);  in igb_fill_rx_descs()
194 		xdp++;  in igb_fill_rx_descs()
207 	struct xdp_buff **xdp;  in igb_alloc_rx_buffers_zc()  local
210 	xdp = &rx_ring->rx_buffer_info_zc[ntu];  in igb_alloc_rx_buffers_zc()
213 		nb_buffs_extra = igb_fill_rx_descs(xsk_pool, xdp, rx_desc,  in igb_alloc_rx_buffers_zc()
220 		xdp = rx_ring->rx_buffer_info_zc;  in igb_alloc_rx_buffers_zc()
225 	nb_buffs = igb_fill_rx_descs(xsk_pool, xdp, rx_desc, count);  in igb_alloc_rx_buffers_zc()
256 		struct xdp_buff *xdp = rx_ring->rx_buffer_info_zc[ntc];  in igb_clean_rx_ring_zc()  local
258 		xsk_buff_free(xdp);  in igb_clean_rx_ring_zc()
266 					    struct xdp_buff *xdp,  in igb_construct_skb_zc()  argument
269 	unsigned int totalsize = xdp->data_end - xdp->data_meta;  in igb_construct_skb_zc()
270 	unsigned int metasize = xdp->data - xdp->data_meta;  in igb_construct_skb_zc()
273 	net_prefetch(xdp->data_meta);  in igb_construct_skb_zc()
283 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,  in igb_construct_skb_zc()
295 			  struct xdp_buff *xdp, struct xsk_buff_pool *xsk_pool,  in igb_run_xdp_zc()  argument
301 	prefetchw(xdp->data_hard_start); /* xdp_frame write */  in igb_run_xdp_zc()
303 	act = bpf_prog_run_xdp(xdp_prog, xdp);  in igb_run_xdp_zc()
306 		err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);  in igb_run_xdp_zc()
322 		result = igb_xdp_xmit_back(adapter, xdp);  in igb_run_xdp_zc()
360 		struct xdp_buff *xdp;  in igb_clean_rx_irq_zc()  local
375 		xdp = rx_ring->rx_buffer_info_zc[ntc];  in igb_clean_rx_irq_zc()
376 		xsk_buff_set_size(xdp, size);  in igb_clean_rx_irq_zc()
377 		xsk_buff_dma_sync_for_cpu(xdp);  in igb_clean_rx_irq_zc()
384 							 xdp->data,  in igb_clean_rx_irq_zc()
387 			xdp->data += ts_hdr_len;  in igb_clean_rx_irq_zc()
388 			xdp->data_meta += ts_hdr_len;  in igb_clean_rx_irq_zc()
392 		xdp_res = igb_run_xdp_zc(adapter, rx_ring, xdp, xsk_pool,  in igb_clean_rx_irq_zc()
402 				xsk_buff_free(xdp);  in igb_clean_rx_irq_zc()
413 		skb = igb_construct_skb_zc(rx_ring, xdp, timestamp);  in igb_clean_rx_irq_zc()
421 		xsk_buff_free(xdp);  in igb_clean_rx_irq_zc()