1 /* SPDX-License-Identifier: GPL-2.0-only */
4  * Copyright 2005-2006 Fen Systems Ltd.
5  * Copyright 2005-2013 Solarflare Communications Inc.
61 /* Checksum generation is a per-queue option in hardware, so each
62  * queue visible to the networking core is backed by two hardware TX
68 #define EFX_TXQ_TYPE_HIGHPRI	4	/* High-priority (for TC) */
70 /* HIGHPRI is Siena-only, and INNER_CSUM is EF10, so no need for both */
83 /* Size of an RX scatter buffer.  Small enough to pack 2 into a 4K page,
86 #define EFX_RX_USR_BUF_SIZE	(2048 - 256)
89  * of every buffer.  Otherwise, we just need to ensure 4-byte
98 /* Non-standard XDP_PACKET_HEADROOM and tailroom to satisfy XDP_REDIRECT and
111  * struct efx_buffer - A general-purpose DMA buffer
126  * struct efx_special_buffer - DMA buffer entered into buffer table
133  * table entries (and so can be physically non-contiguous, although we
146  * struct efx_tx_buffer - buffer state for a TX descriptor
151  * @option: When @flags & %EFX_TX_BUF_OPTION, an EF10-specific option
156  *	This field is zero when the queue slot is empty.
183  * struct efx_tx_queue - An Efx TX queue
196  * @queue: DMA queue number
198  *	Is our index within @channel->tx_queue array.
199  * @type: configuration type of this TX queue.  A bitmask of %EFX_TXQ_TYPE_* flags.
200  * @tso_version: Version of TSO in use for this queue.
203  * @core_txq: The networking core TX queue structure
206  *	%EFX_TX_CB_ORDER into %EFX_TX_CB_SIZE-sized chunks.
209  * @piobuf: PIO buffer region for this TX queue (shared with its partner).
211  * @initialised: Has hardware queue been initialised?
213  * @xdp_tx: Is this an XDP tx queue?
218  *	only get the up-to-date value of @write_count if this
219  *	variable indicates that the queue is empty.  This is to
220  *	avoid cache-line ping-pong between the xmit path and the
235  *	Filled in iff @efx->type->option_descriptors; only used for PIO.
239  *	only get the up-to-date value of read_count if this
240  *	variable indicates that the queue is full.  This is to
241  *	avoid cache-line ping-pong between the xmit path and the
253  * @empty_read_count: If the completion path has seen the queue as empty
255  *	@read_count bitwise-added to %EFX_EMPTY_COUNT_VALID; otherwise 0.
260 	unsigned int queue;  member
310 #define EFX_TX_CB_SIZE	(1 << EFX_TX_CB_ORDER) - NET_IP_ALIGN
313  * struct efx_rx_buffer - An Efx RX data buffer
339  * struct efx_rx_page_state - Page-based rx buffer state
342  * Used to facilitate sharing dma mappings between recycled rx buffers
354  * struct efx_rx_queue - An Efx RX queue
356  * @core_index:  Index of network core RX queue.  Will be >= 0 iff this
357  *	is associated with a real RX queue.
362  * @flush_pending: Set when a RX flush is pending. Has the same lifetime as
364  * @added_count: Number of buffers added to the receive queue.
366  * @removed_count: Number of buffers removed from the receive queue.
377  * @page_ptr_mask: The number of pages in the RX recycle ring minus 1.
378  * @max_fill: RX descriptor maximum fill level (<= ring size)
379  * @fast_fill_trigger: RX descriptor fill level that will trigger a fast fill
381  * @min_fill: RX descriptor minimum non-zero fill level.
384  * @recycle_count: RX buffer recycle counter.
386  * @xdp_rxq_info: XDP specific RX queue information.
431  * struct efx_channel - An Efx channel
433  * A channel comprises an event queue, at least one TX queue, at least
434  * one RX queue, and an associated tasklet for processing the event
435  * queue.
440  * @eventq_init: Event queue initialised flag
442  * @irq: IRQ number (MSI and MSI-X only)
448  * @eventq: Event queue buffer
449  * @eventq_mask: Event queue pointer mask
450  * @eventq_read_ptr: Event queue read pointer
465  * @n_rx_ip_hdr_chksum_err: Count of RX IP header checksum errors
466  * @n_rx_tcp_udp_chksum_err: Count of RX TCP and UDP checksum errors
470  * @n_skbuff_leaks: Count of skbuffs leaked due to RX overrun
471  * @n_rx_nodesc_trunc: Number of RX packets truncated and then dropped due to
473  * @n_rx_merge_events: Number of RX merged completion events
474  * @n_rx_merge_packets: Number of RX packets completed by merged events
475  * @n_rx_xdp_drops: Count of RX packets intentionally dropped due to XDP
476  * @n_rx_xdp_bad_drops: Count of RX packets dropped due to XDP errors
477  * @n_rx_xdp_tx: Count of RX packets retransmitted due to XDP
478  * @n_rx_xdp_redirect: Count of RX packets redirected to a different NIC by XDP
483  * @rx_list: list of SKBs from current RX, awaiting processing
484  * @rx_queue: RX queue for this channel
557  * struct efx_msi_context - Context for each MSI
572  * struct efx_channel_type - distinguishes traffic and extra channels
583  * @keep_eventq: Flag for whether event queue should be kept initialised
612 	STRING_TABLE_LOOKUP((efx)->loopback_mode, efx_siena_loopback_mode)
621 #define EFX_INT_MODE_USE_MSI(x) (((x)->interrupt_mode) <= EFX_INT_MODE_MSI)
633 /* Pseudo bit-mask flow control field */
639  * struct efx_link_state - Current state of the link
641  * @fd: Link is full-duplex
655 	return left->up == right->up && left->fd == right->fd &&  in efx_link_state_equal()
656 		left->fc == right->fc && left->speed == right->speed;  in efx_link_state_equal()
660  * enum efx_phy_mode - PHY operating mode flags
681  * struct efx_hw_stat_desc - Description of a hardware statistic
684  * @dma_width: Width in bits (0 for non-DMA statistics)
685  * @offset: Offset within stats (ignored for non-DMA statistics)
696 /* Number of (single-bit) entries in a multicast filter hash */
710  * struct efx_rss_context - An RSS context for filtering
712  * @rx_hash_udp_4tuple: UDP 4-tuple hashing enabled
727 #define EFX_ARFS_FILTER_ID_PENDING	-1
728 #define EFX_ARFS_FILTER_ID_ERROR	-2
729 #define EFX_ARFS_FILTER_ID_REMOVING	-3
731  * struct efx_arfs_rule - record of an ARFS filter and its IDs
733  * @spec: details of the filter (used as key for hash table).  Use efx->type to
754  * struct efx_async_filter_insertion - Request to asynchronously insert a filter
760  * @flow_id: Identifies the kernel-side flow for which this request was made
776 	EFX_XDP_TX_QUEUES_DEDICATED,	/* one queue per core, locking not needed */
777 	EFX_XDP_TX_QUEUES_SHARED,	/* each queue used by more than 1 core */
782  * struct efx_nic - an Efx NIC
800  * @vi_stride: step between per-VI registers / memory regions
804  * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
806  * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
807  * @irq_rx_moderation_us: IRQ moderation time for RX event queues
812  * @rx_queue: RX DMA queues
815  * @extra_channel_types: Types of extra (non-traffic) channels that
822  * @txq_stop_thresh: TX queue fill level at or above which we stop it.
823  * @txq_wake_thresh: TX queue fill level at or below which we wake it.
824  * @tx_dc_base: Base qword address in SRAM of TX queue descriptor caches
825  * @rx_dc_base: Base qword address in SRAM of RX queue descriptor caches
829  * @n_rx_channels: Number of channels used for RX (= number of RX queues)
836  * @rx_ip_align: RX DMA address offset to have IP header aligned in
838  * @rx_dma_len: Current maximum RX DMA length
839  * @rx_buffer_order: Order (log2) of number of pages for each RX buffer
840  * @rx_buffer_truesize: Amortised allocation size of an RX buffer,
842  * @rx_prefix_size: Size of RX prefix before packet data
843  * @rx_packet_hash_offset: Offset of RX flow hash from start of packet data
845  * @rx_packet_len_offset: Offset of RX packet length from start of packet data
848  *	(valid only if channel->sync_timestamps_enabled; always negative)
855  * @irq_soft_enabled: Are IRQs soft-enabled? If not, IRQ handler will
858  * @irq_zero_count: Number of legacy IRQs seen with queue flags == 0
859  * @irq_level: IRQ level/index for IRQs not triggered by an event queue
860  * @selftest_work: Work item for asynchronous self-test
863  * @mcdi: Management-Controller-to-Driver Interface state
878  * @phy_data: PHY private data (including PHY-specific stats)
887  * @unicast_filter: Flag for Falcon-arch simple unicast filter.
889  * @multicast_hash: Multicast hash table for Falcon-arch.
892  * @fc_disable: When non-zero flow control is disabled. Typically used to
893  *	ensure that network back pressure doesn't delay dma queue flushes.
898  * @loopback_selftest: Offline self-test private state
901  * @filter_state: Architecture-dependent filter table state
903  * @rps_slot_map: bitmap of in-flight entries in @rps_slot
909  * @active_queues: Count of RX and TX queues that haven't been flushed and drained.
912  * @rxq_flush_outstanding: Count of number of RX flushes started but not yet
915  * @flush_wq: wait queue used by efx_nic_flush_queues() to wait for flush completions.
922  * @xdp_rxq_info_failed: Have any of the rx queues failed to initialise their
934  * @n_rx_noskb_drops: Count of RX packets dropped due to failure to allocate an skb
1120 	return efx->net_dev->reg_state == NETREG_REGISTERED;  in efx_dev_registered()
1125 	return efx->port_num;  in efx_port_num()
1143  * struct efx_nic_type - Efx device type definition
1160  * @fini_dmaq: Flush and finalise DMA queues (RX and TX queues)
1190  *	The SDU length may be any value from 0 up to the protocol-
1200  * @irq_enable_master: Enable IRQs on the NIC.  Each event queue must
1203  * @irq_disable_non_ev: Disable non-event IRQs on the NIC.  Each event
1204  *	queue must be separately disabled before this.
1209  * @tx_probe: Allocate resources for TX queue (and select TXQ type)
1210  * @tx_init: Initialise TX queue on the NIC
1211  * @tx_remove: Free resources for TX queue
1213  * @tx_enqueue: Add an SKB to TX queue
1216  * @rx_probe: Allocate resources for RX queue
1217  * @rx_init: Initialise RX queue on the NIC
1218  * @rx_remove: Free resources for RX queue
1219  * @rx_write: Write RX descriptors and doorbell
1221  * @rx_packet: Receive the queued RX buffer on a channel
1222  * @rx_buf_hash_valid: Determine whether the RX prefix contains a valid hash
1223  * @ev_probe: Allocate resources for event queue
1224  * @ev_init: Initialise event queue on the NIC
1225  * @ev_fini: Deinitialise event queue on the NIC
1226  * @ev_remove: Free resources for event queue
1227  * @ev_process: Process events for a queue, up to the given NAPI quota
1228  * @ev_read_ack: Acknowledge read events on a queue, rearming its IRQ
1233  * @filter_update_rx_scatter: Update filters after change to rx scatter setting
1237  * @filter_clear_rx: Remove all RX filters whose priority is less than or
1241  * @filter_get_rx_ids: Get list of RX filters at a given priority
1251  * @mtd_sync: Wait for write-back to complete on MTD partition.  This
1255  * @ptp_set_ts_sync_events: Enable or disable sync events for inline RX
1262  * @tso_versions: Returns mask of firmware-assisted TSO versions supported.
1266  * @print_additional_fwver: Dump NIC-specific additional FW version info
1268  * @rx_recycle_ring_size: Size of the RX recycle ring
1271  * @rxd_ptr_tbl_base: RX descriptor ring base address
1273  * @evq_ptr_tbl_base: Event queue pointer table base address
1274  * @evq_rptr_tbl_base: Event queue read-pointer table base address
1276  * @rx_prefix_size: Size of RX prefix before packet data
1277  * @rx_hash_offset: Offset of RX flow hash within prefix
1279  * @rx_buffer_padding: Size of padding at end of RX packet
1474 	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_channels);  in efx_get_channel()
1475 	return efx->channel[index];  in efx_get_channel()
1480 	for (_channel = (_efx)->channel[0];				\
1482 	     _channel = (_channel->channel + 1 < (_efx)->n_channels) ?	\
1483 		     (_efx)->channel[_channel->channel + 1] : NULL)
1487 	for (_channel = (_efx)->channel[(_efx)->n_channels - 1];	\
1489 	     _channel = _channel->channel ?				\
1490 		     (_efx)->channel[_channel->channel - 1] : NULL)
1495 	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_tx_channels);  in efx_get_tx_channel()
1496 	return efx->channel[efx->tx_channel_offset + index];  in efx_get_tx_channel()
1502 	EFX_WARN_ON_ONCE_PARANOID(index >= efx->n_xdp_channels);  in efx_get_xdp_channel()
1503 	return efx->channel[efx->xdp_channel_offset + index];  in efx_get_xdp_channel()
1508 	return channel->channel - channel->efx->xdp_channel_offset <  in efx_channel_is_xdp_tx()
1509 	       channel->efx->n_xdp_channels;  in efx_channel_is_xdp_tx()
1514 	return channel && channel->channel >= channel->efx->tx_channel_offset;  in efx_channel_has_tx_queues()
1520 		return channel->efx->xdp_tx_per_channel;  in efx_channel_num_tx_queues()
1521 	return channel->efx->tx_queues_per_channel;  in efx_channel_num_tx_queues()
1528 	return channel->tx_queue_by_type[type];  in efx_channel_get_tx_queue()
1544 		for (_tx_queue = (_channel)->tx_queue;			\
1545 		     _tx_queue < (_channel)->tx_queue +			\
1551 	return channel->rx_queue.core_index >= 0;  in efx_channel_has_rx_queue()
1558 	return &channel->rx_queue;  in efx_channel_get_rx_queue()
1561 /* Iterate over all RX queues belonging to a channel */
1566 		for (_rx_queue = &(_channel)->rx_queue;			\
1578 	return efx_rx_queue_channel(rx_queue)->channel;  in efx_rx_queue_index()
1581 /* Returns a pointer to the specified receive buffer in the RX
1582  * descriptor queue.
1587 	return &rx_queue->buffer[index];  in efx_rx_buffer()
1593 	if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask)))  in efx_rx_buf_next()
1600  * EFX_MAX_FRAME_LEN - calculate maximum frame length
1608  * The 10G MAC requires 8-byte alignment on the frame
1611  * Re-clocking by the XGXS on RX can reduce an IPG to 32 bits (half an
1622 	return skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP;  in efx_xmit_with_hwtstamp()
1626 	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;  in efx_xmit_hwtstamp_pending()
1638 				 tx_queue->insert_count - tx_queue->read_count);  in efx_channel_tx_fill_level()
1652 				 tx_queue->insert_count - tx_queue->old_read_count);  in efx_channel_tx_old_fill_level()
1664 	const struct net_device *net_dev = efx->net_dev;  in efx_supported_features()
1666 	return net_dev->features | net_dev->hw_features;  in efx_supported_features()
1669 /* Get the current TX queue insert index. */
1673 	return tx_queue->insert_count & tx_queue->ptr_mask;  in efx_tx_queue_get_insert_index()
1680 	return &tx_queue->buffer[efx_tx_queue_get_insert_index(tx_queue)];  in __efx_tx_queue_get_insert_buffer()
1690 	EFX_WARN_ON_ONCE_PARANOID(buffer->len);  in efx_tx_queue_get_insert_buffer()
1691 	EFX_WARN_ON_ONCE_PARANOID(buffer->flags);  in efx_tx_queue_get_insert_buffer()
1692 	EFX_WARN_ON_ONCE_PARANOID(buffer->unmap_len);  in efx_tx_queue_get_insert_buffer()