6 #include <net/xdp.h>
340  * ice_realloc_zc_buf - reallocate XDP ZC queue pairs
345  * XDP requires more memory, than rx_buf provides.
426  * @xdp: SW ring of xdp_buff that will hold the buffers
437 static u16 ice_fill_rx_descs(struct xsk_buff_pool *pool, struct xdp_buff **xdp,  in ice_fill_rx_descs()  argument
444 	buffs = xsk_buff_alloc_batch(pool, xdp, count);  in ice_fill_rx_descs()
446 		dma = xsk_buff_xdp_get_dma(*xdp);  in ice_fill_rx_descs()
453 		ice_xdp_meta_set_desc(*xdp, rx_desc);  in ice_fill_rx_descs()
456 		xdp++;  in ice_fill_rx_descs()
481 	struct xdp_buff **xdp;  in __ice_alloc_rx_bufs_zc()  local
484 	xdp = ice_xdp_buf(rx_ring, ntu);  in __ice_alloc_rx_bufs_zc()
487 		nb_buffs_extra = ice_fill_rx_descs(xsk_pool, xdp, rx_desc,  in __ice_alloc_rx_bufs_zc()
494 		xdp = ice_xdp_buf(rx_ring, 0);  in __ice_alloc_rx_bufs_zc()
500 	nb_buffs = ice_fill_rx_descs(xsk_pool, xdp, rx_desc, count);  in __ice_alloc_rx_bufs_zc()
542  * @xdp: Pointer to XDP buffer
549 ice_construct_skb_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)  in ice_construct_skb_zc()  argument
551 	unsigned int totalsize = xdp->data_end - xdp->data_meta;  in ice_construct_skb_zc()
552 	unsigned int metasize = xdp->data - xdp->data_meta;  in ice_construct_skb_zc()
557 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_construct_skb_zc()
558 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_construct_skb_zc()
561 	net_prefetch(xdp->data_meta);  in ice_construct_skb_zc()
567 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,  in ice_construct_skb_zc()
575 	if (likely(!xdp_buff_has_frags(xdp)))  in ice_construct_skb_zc()
598 	xsk_buff_free(xdp);  in ice_construct_skb_zc()
604  * @xdp_ring: XDP Tx ring
643 			xsk_buff_free(tx_buf->xdp);  in ice_clean_xdp_irq_zc()
666  * @xdp: XDP buffer to xmit
667  * @xdp_ring: XDP ring to produce descriptor onto
675  * was not enough space on XDP ring
677 static int ice_xmit_xdp_tx_zc(struct xdp_buff *xdp,  in ice_xmit_xdp_tx_zc()  argument
682 	u32 size = xdp->data_end - xdp->data;  in ice_xmit_xdp_tx_zc()
698 	if (unlikely(xdp_buff_has_frags(xdp))) {  in ice_xmit_xdp_tx_zc()
699 		sinfo = xdp_get_shared_info_from_buff(xdp);  in ice_xmit_xdp_tx_zc()
707 	head = xdp;  in ice_xmit_xdp_tx_zc()
712 		dma = xsk_buff_xdp_get_dma(xdp);  in ice_xmit_xdp_tx_zc()
715 		tx_buf->xdp = xdp;  in ice_xmit_xdp_tx_zc()
731 		xdp = xsk_buff_get_frag(head);  in ice_xmit_xdp_tx_zc()
750  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
752  * @xdp: xdp_buff used as input to the XDP program
753  * @xdp_prog: XDP program to run
760 ice_run_xdp_zc(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,  in ice_run_xdp_zc()  argument
767 	act = bpf_prog_run_xdp(xdp_prog, xdp);  in ice_run_xdp_zc()
770 		err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);  in ice_run_xdp_zc()
784 		result = ice_xmit_xdp_tx_zc(xdp, xdp_ring, xsk_pool);  in ice_run_xdp_zc()
827 	/* ZC patch is enabled only when XDP program is set,  in ice_clean_rx_irq_zc()
839 		struct xdp_buff *xdp;  in ice_clean_rx_irq_zc()  local
859 		xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_clean_rx_irq_zc()
864 		xsk_buff_set_size(xdp, size);  in ice_clean_rx_irq_zc()
865 		xsk_buff_dma_sync_for_cpu(xdp);  in ice_clean_rx_irq_zc()
868 			first = xdp;  in ice_clean_rx_irq_zc()
869 		} else if (likely(size) && !xsk_buff_add_frag(first, xdp)) {  in ice_clean_rx_irq_zc()
953  * @xdp_ring: XDP ring to produce the HW Tx descriptor on
978  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1011  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1034  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
1151 		struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);  in ice_xsk_clean_rx_ring()  local
1153 		xsk_buff_free(xdp);  in ice_xsk_clean_rx_ring()
1161  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its buffer pool queues
1174 			xsk_buff_free(tx_buf->xdp);  in ice_xsk_clean_xdp_ring()