115 			state1 = "SMA1 Tx U.FL1 disabled";  in ice_ptp_update_sma_data()
122 		/* U.FL1 Tx will always enable SMA1 Rx */  in ice_ptp_update_sma_data()
123 		state1 = "SMA1 Rx, U.FL1 Tx";  in ice_ptp_update_sma_data()
131 			state2 = "SMA2 Tx, U.FL2 disabled";  in ice_ptp_update_sma_data()
142 			state2 = "SMA2 Tx, U.FL2 Rx";  in ice_ptp_update_sma_data()
191  * ice_ptp_cfg_tx_interrupt - Configure Tx timestamp interrupt for the device
194  * Program the device to respond appropriately to the Tx timestamp interrupt
220 	/* Configure the Tx timestamp interrupt */  in ice_ptp_cfg_tx_interrupt()
359  * This algorithm works even if the PHC time was updated after a Tx timestamp
360  * was requested, but before the Tx timestamp event was reported from
369  * a second, and (b) discarding any Tx timestamp packet if it has waited for
406  * The Tx and Rx timestamps are 40 bits wide, including 32 bits of nominal
444  * ice_ptp_is_tx_tracker_up - Check if Tx tracker is ready for new timestamps
445  * @tx: the PTP Tx timestamp tracker to check
447  * Check that a given PTP Tx timestamp tracker is up, i.e. that it is ready
450  * Assumes the tx->lock spinlock is already held.
453 ice_ptp_is_tx_tracker_up(struct ice_ptp_tx *tx)  in ice_ptp_is_tx_tracker_up()  argument
455 	lockdep_assert_held(&tx->lock);  in ice_ptp_is_tx_tracker_up()
457 	return tx->init && !tx->calibrating;  in ice_ptp_is_tx_tracker_up()
461  * ice_ptp_req_tx_single_tstamp - Request Tx timestamp for a port from FW
462  * @tx: the PTP Tx timestamp tracker
465 void ice_ptp_req_tx_single_tstamp(struct ice_ptp_tx *tx, u8 idx)  in ice_ptp_req_tx_single_tstamp()  argument
473 	if (!tx->init)  in ice_ptp_req_tx_single_tstamp()
476 	ptp_port = container_of(tx, struct ice_ptp_port, tx);  in ice_ptp_req_tx_single_tstamp()
481 	if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {  in ice_ptp_req_tx_single_tstamp()
482 		/* Count the number of Tx timestamps that timed out */  in ice_ptp_req_tx_single_tstamp()
485 		skb = tx->tstamps[idx].skb;  in ice_ptp_req_tx_single_tstamp()
486 		tx->tstamps[idx].skb = NULL;  in ice_ptp_req_tx_single_tstamp()
487 		clear_bit(idx, tx->in_use);  in ice_ptp_req_tx_single_tstamp()
493 	ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);  in ice_ptp_req_tx_single_tstamp()
503 	tx->last_ll_ts_idx_read = idx;  in ice_ptp_req_tx_single_tstamp()
509  * ice_ptp_complete_tx_single_tstamp - Complete Tx timestamp for a port
510  * @tx: the PTP Tx timestamp tracker
512 void ice_ptp_complete_tx_single_tstamp(struct ice_ptp_tx *tx)  in ice_ptp_complete_tx_single_tstamp()  argument
515 	u8 idx = tx->last_ll_ts_idx_read;  in ice_ptp_complete_tx_single_tstamp()
526 	if (!tx->init || tx->last_ll_ts_idx_read < 0)  in ice_ptp_complete_tx_single_tstamp()
529 	ptp_port = container_of(tx, struct ice_ptp_port, tx);  in ice_ptp_complete_tx_single_tstamp()
534 	ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);  in ice_ptp_complete_tx_single_tstamp()
556 		dev_err(ice_pf_to_dev(pf), "Failed to get the Tx tstamp - FW not ready");  in ice_ptp_complete_tx_single_tstamp()
566 	if (raw_tstamp == tx->tstamps[idx].cached_tstamp)  in ice_ptp_complete_tx_single_tstamp()
569 	tx->tstamps[idx].cached_tstamp = raw_tstamp;  in ice_ptp_complete_tx_single_tstamp()
570 	clear_bit(idx, tx->in_use);  in ice_ptp_complete_tx_single_tstamp()
571 	skb = tx->tstamps[idx].skb;  in ice_ptp_complete_tx_single_tstamp()
572 	tx->tstamps[idx].skb = NULL;  in ice_ptp_complete_tx_single_tstamp()
573 	if (test_and_clear_bit(idx, tx->stale))  in ice_ptp_complete_tx_single_tstamp()
596  * ice_ptp_process_tx_tstamp - Process Tx timestamps for a port
597  * @tx: the PTP Tx timestamp tracker
612  * Note that we do not hold the tracking lock while reading the Tx timestamp.
619  * function, or during teardown when the Tx timestamp tracker is being
623  * If a Tx thread starts a new timestamp, we might not begin processing it
626  * If a Tx thread starts a new timestamp just after this function exits, the
632  * this case, software will set the stale bit for any outstanding Tx
636  * If a Tx packet has been waiting for more than 2 seconds, it is not possible
639  * we detect a Tx timestamp request that has waited for this long we assume
643 static void ice_ptp_process_tx_tstamp(struct ice_ptp_tx *tx)  in ice_ptp_process_tx_tstamp()  argument
654 	ptp_port = container_of(tx, struct ice_ptp_port, tx);  in ice_ptp_process_tx_tstamp()
658 	/* Read the Tx ready status first */  in ice_ptp_process_tx_tstamp()
659 	if (tx->has_ready_bitmap) {  in ice_ptp_process_tx_tstamp()
660 		err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready);  in ice_ptp_process_tx_tstamp()
668 	for_each_set_bit(idx, tx->in_use, tx->len) {  in ice_ptp_process_tx_tstamp()
670 		u8 phy_idx = idx + tx->offset;  in ice_ptp_process_tx_tstamp()
676 		if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {  in ice_ptp_process_tx_tstamp()
679 			/* Count the number of Tx timestamps that timed out */  in ice_ptp_process_tx_tstamp()
691 		if (tx->has_ready_bitmap &&  in ice_ptp_process_tx_tstamp()
699 		ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);  in ice_ptp_process_tx_tstamp()
701 		err = ice_read_phy_tstamp(hw, tx->block, phy_idx, &raw_tstamp);  in ice_ptp_process_tx_tstamp()
705 		ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);  in ice_ptp_process_tx_tstamp()
712 		if (!drop_ts && !tx->has_ready_bitmap &&  in ice_ptp_process_tx_tstamp()
713 		    raw_tstamp == tx->tstamps[idx].cached_tstamp)  in ice_ptp_process_tx_tstamp()
721 		spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_process_tx_tstamp()
722 		if (!tx->has_ready_bitmap && raw_tstamp)  in ice_ptp_process_tx_tstamp()
723 			tx->tstamps[idx].cached_tstamp = raw_tstamp;  in ice_ptp_process_tx_tstamp()
724 		clear_bit(idx, tx->in_use);  in ice_ptp_process_tx_tstamp()
725 		skb = tx->tstamps[idx].skb;  in ice_ptp_process_tx_tstamp()
726 		tx->tstamps[idx].skb = NULL;  in ice_ptp_process_tx_tstamp()
727 		if (test_and_clear_bit(idx, tx->stale))  in ice_ptp_process_tx_tstamp()
729 		spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_process_tx_tstamp()
755  * ice_ptp_tx_tstamp_owner - Process Tx timestamps for all ports on the device
765 		struct ice_ptp_tx *tx = &port->tx;  in ice_ptp_tx_tstamp_owner()  local
767 		if (!tx || !tx->init)  in ice_ptp_tx_tstamp_owner()
770 		ice_ptp_process_tx_tstamp(tx);  in ice_ptp_tx_tstamp_owner()
778 		/* Read the Tx ready status first */  in ice_ptp_tx_tstamp_owner()
790  * ice_ptp_tx_tstamp - Process Tx timestamps for this function.
791  * @tx: Tx tracking structure to initialize
794  * Tx timestamps, or ICE_TX_TSTAMP_WORK_DONE otherwise.
796 static enum ice_tx_tstamp_work ice_ptp_tx_tstamp(struct ice_ptp_tx *tx)  in ice_ptp_tx_tstamp()  argument
801 	if (!tx->init)  in ice_ptp_tx_tstamp()
804 	/* Process the Tx timestamp tracker */  in ice_ptp_tx_tstamp()
805 	ice_ptp_process_tx_tstamp(tx);  in ice_ptp_tx_tstamp()
807 	/* Check if there are outstanding Tx timestamps */  in ice_ptp_tx_tstamp()
808 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_tx_tstamp()
809 	more_timestamps = tx->init && !bitmap_empty(tx->in_use, tx->len);  in ice_ptp_tx_tstamp()
810 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_tx_tstamp()
819  * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps
820  * @tx: Tx tracking structure to initialize
826 ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)  in ice_ptp_alloc_tx_tracker()  argument
831 	tstamps = kcalloc(tx->len, sizeof(*tstamps), GFP_KERNEL);  in ice_ptp_alloc_tx_tracker()
832 	in_use = bitmap_zalloc(tx->len, GFP_KERNEL);  in ice_ptp_alloc_tx_tracker()
833 	stale = bitmap_zalloc(tx->len, GFP_KERNEL);  in ice_ptp_alloc_tx_tracker()
843 	tx->tstamps = tstamps;  in ice_ptp_alloc_tx_tracker()
844 	tx->in_use = in_use;  in ice_ptp_alloc_tx_tracker()
845 	tx->stale = stale;  in ice_ptp_alloc_tx_tracker()
846 	tx->init = 1;  in ice_ptp_alloc_tx_tracker()
847 	tx->last_ll_ts_idx_read = -1;  in ice_ptp_alloc_tx_tracker()
849 	spin_lock_init(&tx->lock);  in ice_ptp_alloc_tx_tracker()
857  * @tx: the tracker to flush
859  * Called during teardown when a Tx tracker is being removed.
862 ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)  in ice_ptp_flush_tx_tracker()  argument
870 	err = ice_get_phy_tx_tstamp_ready(hw, tx->block, &tstamp_ready);  in ice_ptp_flush_tx_tracker()
872 		dev_dbg(ice_pf_to_dev(pf), "Failed to get the Tx tstamp ready bitmap for block %u, err %d\n",  in ice_ptp_flush_tx_tracker()
873 			tx->block, err);  in ice_ptp_flush_tx_tracker()
875 		/* If we fail to read the Tx timestamp ready bitmap just  in ice_ptp_flush_tx_tracker()
881 	for_each_set_bit(idx, tx->in_use, tx->len) {  in ice_ptp_flush_tx_tracker()
882 		u8 phy_idx = idx + tx->offset;  in ice_ptp_flush_tx_tracker()
887 			ice_clear_phy_tstamp(hw, tx->block, phy_idx);  in ice_ptp_flush_tx_tracker()
889 		spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_flush_tx_tracker()
890 		skb = tx->tstamps[idx].skb;  in ice_ptp_flush_tx_tracker()
891 		tx->tstamps[idx].skb = NULL;  in ice_ptp_flush_tx_tracker()
892 		clear_bit(idx, tx->in_use);  in ice_ptp_flush_tx_tracker()
893 		clear_bit(idx, tx->stale);  in ice_ptp_flush_tx_tracker()
894 		spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_flush_tx_tracker()
896 		/* Count the number of Tx timestamps flushed */  in ice_ptp_flush_tx_tracker()
906  * @tx: the tracker to mark
908  * Mark currently outstanding Tx timestamps as stale. This prevents sending
916 ice_ptp_mark_tx_tracker_stale(struct ice_ptp_tx *tx)  in ice_ptp_mark_tx_tracker_stale()  argument
920 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_mark_tx_tracker_stale()
921 	bitmap_or(tx->stale, tx->stale, tx->in_use, tx->len);  in ice_ptp_mark_tx_tracker_stale()
922 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_mark_tx_tracker_stale()
929  * Called by the clock owner to flush all the Tx timestamp trackers associated
938 		ice_ptp_flush_tx_tracker(ptp_port_to_pf(port), &port->tx);  in ice_ptp_flush_all_tx_tracker()
942  * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker
944  * @tx: Tx tracking structure to release
946  * Free memory associated with the Tx timestamp tracker.
949 ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)  in ice_ptp_release_tx_tracker()  argument
953 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_release_tx_tracker()
954 	tx->init = 0;  in ice_ptp_release_tx_tracker()
955 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_release_tx_tracker()
960 	ice_ptp_flush_tx_tracker(pf, tx);  in ice_ptp_release_tx_tracker()
962 	kfree(tx->tstamps);  in ice_ptp_release_tx_tracker()
963 	tx->tstamps = NULL;  in ice_ptp_release_tx_tracker()
965 	bitmap_free(tx->in_use);  in ice_ptp_release_tx_tracker()
966 	tx->in_use = NULL;  in ice_ptp_release_tx_tracker()
968 	bitmap_free(tx->stale);  in ice_ptp_release_tx_tracker()
969 	tx->stale = NULL;  in ice_ptp_release_tx_tracker()
971 	tx->len = 0;  in ice_ptp_release_tx_tracker()
975  * ice_ptp_init_tx_eth56g - Initialize tracking for Tx timestamps
977  * @tx: the Tx tracking structure to initialize
980  * Initialize the Tx timestamp tracker for this port. ETH56G PHYs
983  * Return: 0 for success, -ENOMEM when failed to allocate Tx tracker
985 static int ice_ptp_init_tx_eth56g(struct ice_pf *pf, struct ice_ptp_tx *tx,  in ice_ptp_init_tx_eth56g()  argument
988 	tx->block = port;  in ice_ptp_init_tx_eth56g()
989 	tx->offset = 0;  in ice_ptp_init_tx_eth56g()
990 	tx->len = INDEX_PER_PORT_ETH56G;  in ice_ptp_init_tx_eth56g()
991 	tx->has_ready_bitmap = 1;  in ice_ptp_init_tx_eth56g()
993 	return ice_ptp_alloc_tx_tracker(tx);  in ice_ptp_init_tx_eth56g()
997  * ice_ptp_init_tx_e82x - Initialize tracking for Tx timestamps
999  * @tx: the Tx tracking structure to initialize
1002  * Initialize the Tx timestamp tracker for this port. For generic MAC devices,
1008 ice_ptp_init_tx_e82x(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port)  in ice_ptp_init_tx_e82x()  argument
1010 	tx->block = ICE_GET_QUAD_NUM(port);  in ice_ptp_init_tx_e82x()
1011 	tx->offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT_E82X;  in ice_ptp_init_tx_e82x()
1012 	tx->len = INDEX_PER_PORT_E82X;  in ice_ptp_init_tx_e82x()
1013 	tx->has_ready_bitmap = 1;  in ice_ptp_init_tx_e82x()
1015 	return ice_ptp_alloc_tx_tracker(tx);  in ice_ptp_init_tx_e82x()
1019  * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps
1021  * @tx: the Tx tracking structure to initialize
1023  * Initialize the Tx timestamp tracker for this PF. For E810 devices, each
1027 ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)  in ice_ptp_init_tx_e810()  argument
1029 	tx->block = pf->hw.port_info->lport;  in ice_ptp_init_tx_e810()
1030 	tx->offset = 0;  in ice_ptp_init_tx_e810()
1031 	tx->len = INDEX_PER_PORT_E810;  in ice_ptp_init_tx_e810()
1036 	tx->has_ready_bitmap = 0;  in ice_ptp_init_tx_e810()
1038 	return ice_ptp_alloc_tx_tracker(tx);  in ice_ptp_init_tx_e810()
1111  * such as after a time adjustment. It marks any currently outstanding Tx
1146 	ice_ptp_mark_tx_tracker_stale(&pf->ptp.port.tx);  in ice_ptp_reset_cached_phctime()
1202  * ice_ptp_check_tx_fifo - Check whether Tx FIFO is in an OK state
1203  * @port: PTP port for which Tx FIFO is checked
1229 		dev_err(ice_pf_to_dev(pf), "PTP failed to check port %d Tx FIFO, err %d\n",  in ice_ptp_check_tx_fifo()
1251 			"Port %d Tx FIFO still not empty; resetting quad %d\n",  in ice_ptp_check_tx_fifo()
1262  * ice_ptp_wait_for_offsets - Check for valid Tx and Rx offsets
1265  * Check whether hardware has completed measuring the Tx and Rx offset values
1269  * registers with the calibrated offset values and enable timestamping. The Tx
1273  * This function reschedules itself until both Tx and Rx calibration have
1301 		/* Tx and/or Rx offset not yet configured, try again later */  in ice_ptp_wait_for_offsets()
1380 		/* temporarily disable Tx timestamps while calibrating  in ice_ptp_port_phy_restart()
1383 		spin_lock_irqsave(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
1384 		ptp_port->tx.calibrating = true;  in ice_ptp_port_phy_restart()
1385 		spin_unlock_irqrestore(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
1393 		/* Enable Tx timestamps right away */  in ice_ptp_port_phy_restart()
1394 		spin_lock_irqsave(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
1395 		ptp_port->tx.calibrating = false;  in ice_ptp_port_phy_restart()
1396 		spin_unlock_irqrestore(&ptp_port->tx.lock, flags);  in ice_ptp_port_phy_restart()
2689  * ice_ptp_request_ts - Request an available Tx timestamp index
2690  * @tx: the PTP Tx timestamp tracker to request from
2693 s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)  in ice_ptp_request_ts()  argument
2698 	spin_lock_irqsave(&tx->lock, flags);  in ice_ptp_request_ts()
2701 	if (!ice_ptp_is_tx_tracker_up(tx)) {  in ice_ptp_request_ts()
2702 		spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_request_ts()
2707 	idx = find_next_zero_bit(tx->in_use, tx->len,  in ice_ptp_request_ts()
2708 				 tx->last_ll_ts_idx_read + 1);  in ice_ptp_request_ts()
2709 	if (idx == tx->len)  in ice_ptp_request_ts()
2710 		idx = find_first_zero_bit(tx->in_use, tx->len);  in ice_ptp_request_ts()
2712 	if (idx < tx->len) {  in ice_ptp_request_ts()
2717 		set_bit(idx, tx->in_use);  in ice_ptp_request_ts()
2718 		clear_bit(idx, tx->stale);  in ice_ptp_request_ts()
2719 		tx->tstamps[idx].start = jiffies;  in ice_ptp_request_ts()
2720 		tx->tstamps[idx].skb = skb_get(skb);  in ice_ptp_request_ts()
2725 	spin_unlock_irqrestore(&tx->lock, flags);  in ice_ptp_request_ts()
2730 	if (idx >= tx->len)  in ice_ptp_request_ts()
2733 		return idx + tx->offset;  in ice_ptp_request_ts()
2737  * ice_ptp_process_ts - Process the PTP Tx timestamps
2740  * Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding Tx
2751 		return ice_ptp_tx_tstamp(&pf->ptp.port.tx);  in ice_ptp_process_ts()
2756 		WARN_ONCE(1, "Unexpected Tx timestamp interrupt mode %u\n",  in ice_ptp_process_ts()
2763  * ice_ptp_maybe_trigger_tx_interrupt - Trigger Tx timstamp interrupt
2766  * The device PHY issues Tx timestamp interrupts to the driver for processing
2803 …dev_dbg(dev, "PTP periodic task detected waiting timestamps. Triggering Tx timestamp interrupt now…  in ice_ptp_maybe_trigger_tx_interrupt()
2843 	/* Disable timestamping for both Tx and Rx */  in ice_ptp_prepare_for_reset()
2851 	ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);  in ice_ptp_prepare_for_reset()
3141 		return ice_ptp_init_tx_eth56g(pf, &ptp_port->tx,  in ice_ptp_init_port()
3144 		return ice_ptp_init_tx_e810(pf, &ptp_port->tx);  in ice_ptp_init_port()
3149 		return ice_ptp_init_tx_e82x(pf, &ptp_port->tx,  in ice_ptp_init_port()
3157  * ice_ptp_init_tx_interrupt_mode - Initialize device Tx interrupt mode
3160  * Initialize the Tx timestamp interrupt mode for this device. For most device
3178 		/* other PHY types handle their own Tx interrupt */  in ice_ptp_init_tx_interrupt_mode()
3193  * items used for asynchronous work such as Tx timestamps and periodic work.
3237 	/* Configure initial Tx interrupt settings */  in ice_ptp_init()
3273 	/* Disable timestamping for both Tx and Rx */  in ice_ptp_release()
3278 	ice_ptp_release_tx_tracker(pf, &pf->ptp.port.tx);  in ice_ptp_release()