9  * PTP support is assisted by firmware running on the MC, which provides
11  * PTP event packets are queued onto internal queues for subsequent processing;
17  *	with the hardware timestamp.  The PTP receive packet queue is searched
21  *	It is important for the operation of the PTP protocol that the ordering
47 #include "nic.h" /* indirectly includes ptp.h */
50 /* Maximum number of events expected to make up a PTP event */
80 /* Offsets into PTP packet for identification.  These offsets are from the
81  * start of the IP header, not the MAC header.  Note that neither PTP V1 nor
82  * PTP V2 permit the use of IPV4 options.
92 /* The minimum length of a PTP V1 packet for offsets, etc. to be valid:
103 /* The minimum length of a PTP V2 packet for offsets, etc. to be valid:
172  * struct efx_ptp_event_rx - A PTP receive event (from MC)
174  * @seq0: First part of (PTP) UUID
175  * @seq1: Second part of (PTP) UUID and sequence number
208  * struct efx_ptp_rxfilter - Filter for PTP packets
226  * struct efx_ptp_data - Precision Time Protocol (PTP) state
228  * @channel: The PTP channel (for Medford and Medford2)
231  * @workwq: Work queue for processing pending PTP operations
234  * @reset_required: A serious error has occurred and the PTP task needs to be
236  * @rxfilters_mcast: Receive filters for multicast PTP packets
237  * @rxfilters_ucast: Receive filters for unicast PTP packets
239  * @enabled: PTP operation enabled
240  * @mode: Mode in which PTP operating (PTP version)
256  * @ts_corrections.ptp_tx: Required driver correction of PTP packet transmit
258  * @ts_corrections.ptp_rx: Required driver correction of PTP packet receive
266  * @evt_frags: Partly assembled PTP events
279  * @txbuf: Buffer for use when transmitting (PTP) packets to MC (avoids
354 static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm);
355 static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta);
356 static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts);
357 static int efx_phc_settime(struct ptp_clock_info *ptp,
359 static int efx_phc_enable(struct ptp_clock_info *ptp,
369 /* PTP 'extra' channel is still a traffic channel, but we only create TX queues
370  * if PTP uses MAC TX timestamps, not if PTP uses the MC directly to transmit.
552 				    struct efx_ptp_data *ptp,  in efx_ptp_mac_nic_to_ktime_correction()  argument
596 		kt = ptp->nic_to_kernel_time(nic_major, nic_minor,  in efx_ptp_mac_nic_to_ktime_correction()
605 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_nic_to_kernel_time()  local
609 		kt = efx_ptp_mac_nic_to_ktime_correction(efx, ptp,  in efx_ptp_nic_to_kernel_time()
612 				ptp->ts_corrections.general_tx);  in efx_ptp_nic_to_kernel_time()
614 		kt = ptp->nic_to_kernel_time(  in efx_ptp_nic_to_kernel_time()
617 				ptp->ts_corrections.general_tx);  in efx_ptp_nic_to_kernel_time()
621 /* Get PTP attributes and set up time conversions */
626 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_attributes()  local
631 	/* Get the PTP attributes. If the NIC doesn't support the operation we  in efx_ptp_get_attributes()
644 		pci_info(efx->pci_dev, "no PTP support\n");  in efx_ptp_get_attributes()
654 		ptp->ns_to_nic_time = efx_ptp_ns_to_s27;  in efx_ptp_get_attributes()
655 		ptp->nic_to_kernel_time = efx_ptp_s27_to_ktime_correction;  in efx_ptp_get_attributes()
656 		ptp->nic_time.minor_max = 1 << 27;  in efx_ptp_get_attributes()
657 		ptp->nic_time.sync_event_minor_shift = 19;  in efx_ptp_get_attributes()
660 		ptp->ns_to_nic_time = efx_ptp_ns_to_s_qns;  in efx_ptp_get_attributes()
661 		ptp->nic_to_kernel_time = efx_ptp_s_qns_to_ktime_correction;  in efx_ptp_get_attributes()
662 		ptp->nic_time.minor_max = 4000000000UL;  in efx_ptp_get_attributes()
663 		ptp->nic_time.sync_event_minor_shift = 24;  in efx_ptp_get_attributes()
675 	ptp->nic_time.sync_event_diff_min = ptp->nic_time.minor_max  in efx_ptp_get_attributes()
676 		- (ptp->nic_time.minor_max / 10);  in efx_ptp_get_attributes()
677 	ptp->nic_time.sync_event_diff_max = (ptp->nic_time.minor_max / 4)  in efx_ptp_get_attributes()
678 		+ (ptp->nic_time.minor_max / 10);  in efx_ptp_get_attributes()
689 		ptp->min_synchronisation_ns =  in efx_ptp_get_attributes()
693 		ptp->min_synchronisation_ns = DEFAULT_MIN_SYNCHRONISATION_NS;  in efx_ptp_get_attributes()
697 		ptp->capabilities = MCDI_DWORD(outbuf,  in efx_ptp_get_attributes()
700 		ptp->capabilities = 0;  in efx_ptp_get_attributes()
706 	if (ptp->capabilities & (1 << MC_CMD_PTP_OUT_GET_ATTRIBUTES_FP44_FREQ_ADJ_LBN))  in efx_ptp_get_attributes()
707 		ptp->adjfreq_ppb_shift = PPB_SHIFT_FP44;  in efx_ptp_get_attributes()
709 		ptp->adjfreq_ppb_shift = PPB_SHIFT_FP40;  in efx_ptp_get_attributes()
714 /* Get PTP timestamp corrections */
770 /* Enable MCDI PTP support. */
794 /* Disable MCDI PTP support.
810 	/* If we get ENOSYS, the NIC doesn't support PTP, and thus this function  in efx_ptp_disable()
814 		pci_info(efx->pci_dev, "no PTP support\n");  in efx_ptp_disable()
836 		  "ERROR: PTP requires MSI-X and 1 additional interrupt"  in efx_ptp_handle_no_channel()
837 		  "vector. PTP disabled\n");  in efx_ptp_handle_no_channel()
848 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_send_times()  local
849 	int *mc_running = ptp->start.addr;  in efx_ptp_send_times()
919 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_times()  local
941 			&ptp->timeset[i]);  in efx_ptp_process_times()
944 			ptp->nic_to_kernel_time(0, ptp->timeset[i].wait, 0));  in efx_ptp_process_times()
945 		window = ptp->timeset[i].window;  in efx_ptp_process_times()
958 			++ptp->invalid_sync_windows;  in efx_ptp_process_times()
960 			++ptp->oversize_sync_windows;  in efx_ptp_process_times()
961 		} else if (corrected < ptp->min_synchronisation_ns) {  in efx_ptp_process_times()
962 			++ptp->undersize_sync_windows;  in efx_ptp_process_times()
971 			   "PTP no suitable synchronisations\n");  in efx_ptp_process_times()
981 	start_sec = ptp->timeset[last_good].host_start >> MC_NANOSECOND_BITS;  in efx_ptp_process_times()
986 			   "PTP bad synchronisation seconds\n");  in efx_ptp_process_times()
992 		(ptp->timeset[last_good].host_start & MC_NANOSECOND_MASK);  in efx_ptp_process_times()
998 	mc_time = ptp->nic_to_kernel_time(ptp->timeset[last_good].major,  in efx_ptp_process_times()
999 					  ptp->timeset[last_good].minor, 0);  in efx_ptp_process_times()
1005 	ptp->host_time_pps = *last_time;  in efx_ptp_process_times()
1006 	pps_sub_ts(&ptp->host_time_pps, delta);  in efx_ptp_process_times()
1014 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_synchronize()  local
1021 	int *start = ptp->start.addr;  in efx_ptp_synchronize()
1028 		       ptp->start.dma_addr);  in efx_ptp_synchronize()
1044 		++ptp->fast_syncs;  in efx_ptp_synchronize()
1046 		++ptp->sync_timeouts;  in efx_ptp_synchronize()
1060 			++ptp->good_syncs;  in efx_ptp_synchronize()
1062 			++ptp->no_time_syncs;  in efx_ptp_synchronize()
1069 		++ptp->bad_syncs;  in efx_ptp_synchronize()
1074 /* Transmit a PTP packet via the dedicated hardware timestamped queue. */
1092 		 * Although it is theoretically safe for current PTP TX/RX code  in efx_ptp_xmit_skb_queue()
1096 		 *      1) The code invoked is mainly implemented for non-PTP  in efx_ptp_xmit_skb_queue()
1099 		 *      2) This being a single PTP packet, better to not  in efx_ptp_xmit_skb_queue()
1116 		WARN_ONCE(1, "PTP channel has no timestamped tx queue\n");  in efx_ptp_xmit_skb_queue()
1121 /* Transmit a PTP packet, via the MCDI interface, to the wire. */
1179 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_process_events()  local
1182 	while ((skb = skb_dequeue(&ptp->rxq))) {  in efx_ptp_process_events()
1190 			++ptp->rx_no_timestamp;  in efx_ptp_process_events()
1194 			skb_queue_head(&ptp->rxq, skb);  in efx_ptp_process_events()
1256 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_filter()  local
1281 	queue_delayed_work(ptp->workwq, &ptp->cleanup_work,  in efx_ptp_insert_filter()
1317 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_eth_multicast_filter()  local
1324 	return efx_ptp_insert_filter(efx, &ptp->rxfilters_mcast, &spec, 0);  in efx_ptp_insert_eth_multicast_filter()
1329 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_multicast_filters()  local
1332 	if (!ptp->channel || !list_empty(&ptp->rxfilters_mcast))  in efx_ptp_insert_multicast_filters()
1338 	rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_mcast,  in efx_ptp_insert_multicast_filters()
1344 	rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_mcast,  in efx_ptp_insert_multicast_filters()
1351 	 * PTP over IPv6 and Ethernet  in efx_ptp_insert_multicast_filters()
1354 		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_mcast,  in efx_ptp_insert_multicast_filters()
1359 		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_mcast,  in efx_ptp_insert_multicast_filters()
1374 	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);  in efx_ptp_insert_multicast_filters()
1395 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_insert_unicast_filter()  local
1407 		rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,  in efx_ptp_insert_unicast_filter()
1412 		rc = efx_ptp_insert_ipv4_filter(efx, &ptp->rxfilters_ucast,  in efx_ptp_insert_unicast_filter()
1415 		/* IPv6 PTP only supported by devices with MAC hw timestamp */  in efx_ptp_insert_unicast_filter()
1418 		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,  in efx_ptp_insert_unicast_filter()
1423 		rc = efx_ptp_insert_ipv6_filter(efx, &ptp->rxfilters_ucast,  in efx_ptp_insert_unicast_filter()
1435 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_start()  local
1438 	ptp->reset_required = false;  in efx_ptp_start()
1448 	ptp->evt_frag_idx = 0;  in efx_ptp_start()
1449 	ptp->current_adjfreq = 0;  in efx_ptp_start()
1454 	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);  in efx_ptp_start()
1460 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_stop()  local
1463 	if (ptp == NULL)  in efx_ptp_stop()
1468 	efx_ptp_remove_filters(efx, &ptp->rxfilters_mcast);  in efx_ptp_stop()
1469 	efx_ptp_remove_filters(efx, &ptp->rxfilters_ucast);  in efx_ptp_stop()
1487 	struct efx_ptp_data *ptp =  in efx_ptp_pps_worker()  local
1489 	struct efx_nic *efx = ptp->efx;  in efx_ptp_pps_worker()
1496 	ptp_evt.pps_times = ptp->host_time_pps;  in efx_ptp_pps_worker()
1497 	ptp_clock_event(ptp->phc_clock, &ptp_evt);  in efx_ptp_pps_worker()
1526 	struct efx_ptp_data *ptp =  in efx_ptp_cleanup_worker()  local
1530 	list_for_each_entry_safe(rxfilter, tmp, &ptp->rxfilters_ucast, list) {  in efx_ptp_cleanup_worker()
1532 			efx_ptp_remove_one_filter(ptp->efx, rxfilter);  in efx_ptp_cleanup_worker()
1535 	if (!list_empty(&ptp->rxfilters_ucast)) {  in efx_ptp_cleanup_worker()
1536 		queue_delayed_work(ptp->workwq, &ptp->cleanup_work,  in efx_ptp_cleanup_worker()
1557 /* Initialise PTP state. */
1560 	struct efx_ptp_data *ptp;  in efx_ptp_probe()  local
1568 	ptp = kzalloc(sizeof(struct efx_ptp_data), GFP_KERNEL);  in efx_ptp_probe()
1569 	efx->ptp_data = ptp;  in efx_ptp_probe()
1573 	ptp->efx = efx;  in efx_ptp_probe()
1574 	ptp->channel = channel;  in efx_ptp_probe()
1576 	rc = efx_nic_alloc_buffer(efx, &ptp->start, sizeof(int), GFP_KERNEL);  in efx_ptp_probe()
1580 	skb_queue_head_init(&ptp->rxq);  in efx_ptp_probe()
1581 	skb_queue_head_init(&ptp->txq);  in efx_ptp_probe()
1582 	ptp->workwq = create_singlethread_workqueue("sfc_ptp");  in efx_ptp_probe()
1583 	if (!ptp->workwq) {  in efx_ptp_probe()
1589 		ptp->xmit_skb = efx_ptp_xmit_skb_queue;  in efx_ptp_probe()
1593 		ptp->xmit_skb = efx_ptp_xmit_skb_mc;  in efx_ptp_probe()
1596 	INIT_WORK(&ptp->work, efx_ptp_worker);  in efx_ptp_probe()
1597 	INIT_DELAYED_WORK(&ptp->cleanup_work, efx_ptp_cleanup_worker);  in efx_ptp_probe()
1598 	ptp->config.flags = 0;  in efx_ptp_probe()
1599 	ptp->config.tx_type = HWTSTAMP_TX_OFF;  in efx_ptp_probe()
1600 	ptp->config.rx_filter = HWTSTAMP_FILTER_NONE;  in efx_ptp_probe()
1601 	INIT_LIST_HEAD(&ptp->rxfilters_mcast);  in efx_ptp_probe()
1602 	INIT_LIST_HEAD(&ptp->rxfilters_ucast);  in efx_ptp_probe()
1604 	/* Get the NIC PTP attributes and set up time conversions */  in efx_ptp_probe()
1616 		ptp->phc_clock_info = efx_phc_clock_info;  in efx_ptp_probe()
1617 		ptp->phc_clock = ptp_clock_register(&ptp->phc_clock_info,  in efx_ptp_probe()
1619 		if (IS_ERR(ptp->phc_clock)) {  in efx_ptp_probe()
1620 			rc = PTR_ERR(ptp->phc_clock);  in efx_ptp_probe()
1622 		} else if (ptp->phc_clock) {  in efx_ptp_probe()
1623 			INIT_WORK(&ptp->pps_work, efx_ptp_pps_worker);  in efx_ptp_probe()
1624 			ptp->pps_workwq = create_singlethread_workqueue("sfc_pps");  in efx_ptp_probe()
1625 			if (!ptp->pps_workwq) {  in efx_ptp_probe()
1631 	ptp->nic_ts_enabled = false;  in efx_ptp_probe()
1641 	efx_nic_free_buffer(efx, &ptp->start);  in efx_ptp_probe()
1650 /* Initialise PTP channel.
1653  * overlap with 'rxq0' because ptp.c doesn't use skb_record_rx_queue.
1664 	/* Failure to probe PTP is not fatal; this channel will just not be  in efx_ptp_probe_channel()
1671 			   "Failed to probe PTP, rc=%d\n", rc);  in efx_ptp_probe_channel()
1710 	snprintf(buf, len, "%s-ptp", channel->efx->name);  in efx_ptp_get_channel_name()
1713 /* Determine whether this packet should be processed by the PTP module
1731 /* Receive a PTP packet.  Packets are queued until the arrival of
1738 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_rx()  local
1746 	if (ptp->mode == MC_CMD_PTP_MODE_V1) {  in efx_ptp_rx()
1771 		 * the packet for PTP V1 and V2  in efx_ptp_rx()
1779 	skb_queue_tail(&ptp->rxq, skb);  in efx_ptp_rx()
1780 	queue_work(ptp->workwq, &ptp->work);  in efx_ptp_rx()
1785 /* Transmit a PTP packet.  This has to be transmitted by the MC
1791 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_tx()  local
1793 	skb_queue_tail(&ptp->txq, skb);  in efx_ptp_tx()
1798 	queue_work(ptp->workwq, &ptp->work);  in efx_ptp_tx()
1864 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_get_ts_info()  local
1869 	if (!ptp)  in efx_ptp_get_ts_info()
1890 	ts_info->rx_filters = ptp->efx->type->hwtstamp_filters;  in efx_ptp_get_ts_info()
1897 	/* Not a PTP enabled port */  in efx_ptp_set_ts_config()
1907 	/* Not a PTP enabled port */  in efx_ptp_get_ts_config()
1916 	struct efx_ptp_data *ptp = efx->ptp_data;  in ptp_event_failure()  local
1919 		"PTP unexpected event length: got %d expected %d\n",  in ptp_event_failure()
1920 		ptp->evt_frag_idx, expected_frag_len);  in ptp_event_failure()
1921 	ptp->reset_required = true;  in ptp_event_failure()
1922 	queue_work(ptp->workwq, &ptp->work);  in ptp_event_failure()
1925 static void ptp_event_fault(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_fault()  argument
1927 	int code = EFX_QWORD_FIELD(ptp->evt_frags[0], MCDI_EVENT_DATA);  in ptp_event_fault()
1928 	if (ptp->evt_frag_idx != 1) {  in ptp_event_fault()
1933 	netif_err(efx, hw, efx->net_dev, "PTP error %d\n", code);  in ptp_event_fault()
1936 static void ptp_event_pps(struct efx_nic *efx, struct efx_ptp_data *ptp)  in ptp_event_pps()  argument
1938 	if (ptp->nic_ts_enabled)  in ptp_event_pps()
1939 		queue_work(ptp->pps_workwq, &ptp->pps_work);  in ptp_event_pps()
1944 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_ptp_event()  local
1947 	if (!ptp) {  in efx_ptp_event()
1950 				   "Received PTP event but PTP not set up\n");  in efx_ptp_event()
1956 	if (!ptp->enabled)  in efx_ptp_event()
1959 	if (ptp->evt_frag_idx == 0) {  in efx_ptp_event()
1960 		ptp->evt_code = code;  in efx_ptp_event()
1961 	} else if (ptp->evt_code != code) {  in efx_ptp_event()
1963 			  "PTP out of sequence event %d\n", code);  in efx_ptp_event()
1964 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1967 	ptp->evt_frags[ptp->evt_frag_idx++] = *ev;  in efx_ptp_event()
1972 			ptp_event_fault(efx, ptp);  in efx_ptp_event()
1975 			ptp_event_pps(efx, ptp);  in efx_ptp_event()
1979 				  "PTP unknown event %d\n", code);  in efx_ptp_event()
1982 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1983 	} else if (MAX_EVENT_FRAGS == ptp->evt_frag_idx) {  in efx_ptp_event()
1985 			  "PTP too many event fragments\n");  in efx_ptp_event()
1986 		ptp->evt_frag_idx = 0;  in efx_ptp_event()
1993 	struct efx_ptp_data *ptp = efx->ptp_data;  in efx_time_sync_event()  local
2004 			<< ptp->nic_time.sync_event_minor_shift;  in efx_time_sync_event()
2030 	struct efx_ptp_data *ptp = efx->ptp_data;  in __efx_rx_skb_attach_timestamp()  local
2045 		diff += ptp->nic_time.minor_max;  in __efx_rx_skb_attach_timestamp()
2048 	carry = (channel->sync_timestamp_minor >= ptp->nic_time.minor_max - diff) ?  in __efx_rx_skb_attach_timestamp()
2051 	if (diff <= ptp->nic_time.sync_event_diff_max) {  in __efx_rx_skb_attach_timestamp()
2056 	} else if (diff >= ptp->nic_time.sync_event_diff_min) {  in __efx_rx_skb_attach_timestamp()
2078 		ptp->nic_to_kernel_time(pkt_timestamp_major,  in __efx_rx_skb_attach_timestamp()
2080 					ptp->ts_corrections.general_rx);  in __efx_rx_skb_attach_timestamp()
2083 static int efx_phc_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)  in efx_phc_adjfine()  argument
2085 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_adjfine()
2118 static int efx_phc_adjtime(struct ptp_clock_info *ptp, s64 delta)  in efx_phc_adjtime()  argument
2121 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_adjtime()
2138 static int efx_phc_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)  in efx_phc_gettime()  argument
2140 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_gettime()
2164 static int efx_phc_settime(struct ptp_clock_info *ptp,  in efx_phc_settime()  argument
2175 	rc = efx_phc_gettime(ptp, &time_now);  in efx_phc_settime()
2181 	rc = efx_phc_adjtime(ptp, timespec64_to_ns(&delta));  in efx_phc_settime()
2188 static int efx_phc_enable(struct ptp_clock_info *ptp,  in efx_phc_enable()  argument
2192 	struct efx_ptp_data *ptp_data = container_of(ptp,  in efx_phc_enable()
2215 	/* Check whether PTP is implemented on this NIC.  The DISABLE  in efx_ptp_defer_probe_with_channel()
2226 		netif_err(efx, drv, efx->net_dev, "Failed to restart PTP.\n");  in efx_ptp_start_datapath()