1	// SPDX-License-Identifier: GPL-2.0+
2	/* Copyright (C) 2011 Richard Cochran <richardcochran@gmail.com> */
3
4	#include <linux/module.h>
5	#include <linux/device.h>
6	#include <linux/pci.h>
7	#include <linux/ptp_classify.h>
8
9	#include "igb.h"
10
11	#define INCVALUE_MASK 0x7fffffff
12	#define ISGN 0x80000000
13
14	/* The 82580 timesync updates the system timer every 8ns by 8ns,
15	* and this update value cannot be reprogrammed.
16	*
17	* Neither the 82576 nor the 82580 offer registers wide enough to hold
18	* nanoseconds time values for very long. For the 82580, SYSTIM always
19	* counts nanoseconds, but the upper 24 bits are not available. The
20	* frequency is adjusted by changing the 32 bit fractional nanoseconds
21	* register, TIMINCA.
22	*
23	* For the 82576, the SYSTIM register time unit is affect by the
24	* choice of the 24 bit TININCA:IV (incvalue) field. Five bits of this
25	* field are needed to provide the nominal 16 nanosecond period,
26	* leaving 19 bits for fractional nanoseconds.
27	*
28	* We scale the NIC clock cycle by a large factor so that relatively
29	* small clock corrections can be added or subtracted at each clock
30	* tick. The drawbacks of a large factor are a) that the clock
31	* register overflows more quickly (not such a big deal) and b) that
32	* the increment per tick has to fit into 24 bits. As a result we
33	* need to use a shift of 19 so we can fit a value of 16 into the
34	* TIMINCA register.
35	*
36	*
37	* SYSTIMH SYSTIML
38	* +--------------+ +---+---+------+
39	* 82576 | 32 | | 8 | 5 | 19 |
40	* +--------------+ +---+---+------+
41	* \________ 45 bits _______/ fract
42	*
43	* +----------+---+ +--------------+
44	* 82580 | 24 | 8 | | 32 |
45	* +----------+---+ +--------------+
46	* reserved \______ 40 bits _____/
47	*
48	*
49	* The 45 bit 82576 SYSTIM overflows every
50	* 2^45 * 10^-9 / 3600 = 9.77 hours.
51	*
52	* The 40 bit 82580 SYSTIM overflows every
53	* 2^40 * 10^-9 / 60 = 18.3 minutes.
54	*
55	* SYSTIM is converted to real time using a timecounter. As
56	* timecounter_cyc2time() allows old timestamps, the timecounter needs
57	* to be updated at least once per half of the SYSTIM interval.
58	* Scheduling of delayed work is not very accurate, and also the NIC
59	* clock can be adjusted to run up to 6% faster and the system clock
60	* up to 10% slower, so we aim for 6 minutes to be sure the actual
61	* interval in the NIC time is shorter than 9.16 minutes.
62	*/
63
64	#define IGB_SYSTIM_OVERFLOW_PERIOD (HZ * 60 * 6)
65	#define IGB_PTP_TX_TIMEOUT (HZ * 15)
66	#define INCPERIOD_82576 BIT(E1000_TIMINCA_16NS_SHIFT)
67	#define INCVALUE_82576_MASK GENMASK(E1000_TIMINCA_16NS_SHIFT - 1, 0)
68	#define INCVALUE_82576 (16u << IGB_82576_TSYNC_SHIFT)
69	#define IGB_NBITS_82580 40
70	#define IGB_82580_BASE_PERIOD 0x800000000
71
72	static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter);
73	static void igb_ptp_sdp_init(struct igb_adapter *adapter);
74
75	/* SYSTIM read access for the 82576 */
76	static u64 igb_ptp_read_82576(const struct cyclecounter *cc)
77	{
78	struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc);
79	struct e1000_hw *hw = &igb->hw;
80	u64 val;
81	u32 lo, hi;
82
83	lo = rd32(E1000_SYSTIML);
84	hi = rd32(E1000_SYSTIMH);
85
86	val = ((u64) hi) << 32;
87	val |= lo;
88
89	return val;
90	}
91
92	/* SYSTIM read access for the 82580 */
93	static u64 igb_ptp_read_82580(const struct cyclecounter *cc)
94	{
95	struct igb_adapter *igb = container_of(cc, struct igb_adapter, cc);
96	struct e1000_hw *hw = &igb->hw;
97	u32 lo, hi;
98	u64 val;
99
100	/* The timestamp latches on lowest register read. For the 82580
101	* the lowest register is SYSTIMR instead of SYSTIML. However we only
102	* need to provide nanosecond resolution, so we just ignore it.
103	*/
104	rd32(E1000_SYSTIMR);
105	lo = rd32(E1000_SYSTIML);
106	hi = rd32(E1000_SYSTIMH);
107
108	val = ((u64) hi) << 32;
109	val |= lo;
110
111	return val;
112	}
113
114	/* SYSTIM read access for I210/I211 */
115	static void igb_ptp_read_i210(struct igb_adapter *adapter,
116	struct timespec64 *ts)
117	{
118	struct e1000_hw *hw = &adapter->hw;
119	u32 sec, nsec;
120
121	/* The timestamp latches on lowest register read. For I210/I211, the
122	* lowest register is SYSTIMR. Since we only need to provide nanosecond
123	* resolution, we can ignore it.
124	*/
125	rd32(E1000_SYSTIMR);
126	nsec = rd32(E1000_SYSTIML);
127	sec = rd32(E1000_SYSTIMH);
128
129	ts->tv_sec = sec;
130	ts->tv_nsec = nsec;
131	}
132
133	static void igb_ptp_write_i210(struct igb_adapter *adapter,
134	const struct timespec64 *ts)
135	{
136	struct e1000_hw *hw = &adapter->hw;
137
138	/* Writing the SYSTIMR register is not necessary as it only provides
139	* sub-nanosecond resolution.
140	*/
141	wr32(E1000_SYSTIML, ts->tv_nsec);
142	wr32(E1000_SYSTIMH, (u32)ts->tv_sec);
143	}
144
145	/**
146	* igb_ptp_systim_to_hwtstamp - convert system time value to hw timestamp
147	* @adapter: board private structure
148	* @hwtstamps: timestamp structure to update
149	* @systim: unsigned 64bit system time value.
150	*
151	* We need to convert the system time value stored in the RX/TXSTMP registers
152	* into a hwtstamp which can be used by the upper level timestamping functions.
153	*
154	* The 'tmreg_lock' spinlock is used to protect the consistency of the
155	* system time value. This is needed because reading the 64 bit time
156	* value involves reading two (or three) 32 bit registers. The first
157	* read latches the value. Ditto for writing.
158	*
159	* In addition, here have extended the system time with an overflow
160	* counter in software.
161	**/
162	static void igb_ptp_systim_to_hwtstamp(struct igb_adapter *adapter,
163	struct skb_shared_hwtstamps *hwtstamps,
164	u64 systim)
165	{
166	unsigned long flags;
167	u64 ns;
168
169	memset(hwtstamps, 0, sizeof(*hwtstamps));
170
171	switch (adapter->hw.mac.type) {
172	case e1000_82576:
173	case e1000_82580:
174	case e1000_i354:
175	case e1000_i350:
176	spin_lock_irqsave(&adapter->tmreg_lock, flags);
177	ns = timecounter_cyc2time(tc: &adapter->tc, cycle_tstamp: systim);
178	spin_unlock_irqrestore(lock: &adapter->tmreg_lock, flags);
179
180	hwtstamps->hwtstamp = ns_to_ktime(ns);
181	break;
182	case e1000_i210:
183	case e1000_i211:
184	/* Upper 32 bits contain s, lower 32 bits contain ns. */
185	hwtstamps->hwtstamp = ktime_set(secs: systim >> 32,
186	nsecs: systim & 0xFFFFFFFF);
187	break;
188	default:
189	break;
190	}
191	}
192
193	/* PTP clock operations */
194	static int igb_ptp_adjfine_82576(struct ptp_clock_info *ptp, long scaled_ppm)
195	{
196	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
197	ptp_caps);
198	struct e1000_hw *hw = &igb->hw;
199	u64 incvalue;
200
201	incvalue = adjust_by_scaled_ppm(INCVALUE_82576, scaled_ppm);
202
203	wr32(E1000_TIMINCA, INCPERIOD_82576 | (incvalue & INCVALUE_82576_MASK));
204
205	return 0;
206	}
207
208	static int igb_ptp_adjfine_82580(struct ptp_clock_info *ptp, long scaled_ppm)
209	{
210	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
211	ptp_caps);
212	struct e1000_hw *hw = &igb->hw;
213	bool neg_adj;
214	u64 rate;
215	u32 inca;
216
217	neg_adj = diff_by_scaled_ppm(IGB_82580_BASE_PERIOD, scaled_ppm, diff: &rate);
218
219	inca = rate & INCVALUE_MASK;
220	if (neg_adj)
221	inca |= ISGN;
222
223	wr32(E1000_TIMINCA, inca);
224
225	return 0;
226	}
227
228	static int igb_ptp_adjtime_82576(struct ptp_clock_info *ptp, s64 delta)
229	{
230	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
231	ptp_caps);
232	unsigned long flags;
233
234	spin_lock_irqsave(&igb->tmreg_lock, flags);
235	timecounter_adjtime(tc: &igb->tc, delta);
236	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
237
238	return 0;
239	}
240
241	static int igb_ptp_adjtime_i210(struct ptp_clock_info *ptp, s64 delta)
242	{
243	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
244	ptp_caps);
245	unsigned long flags;
246	struct timespec64 now, then = ns_to_timespec64(nsec: delta);
247
248	spin_lock_irqsave(&igb->tmreg_lock, flags);
249
250	igb_ptp_read_i210(adapter: igb, ts: &now);
251	now = timespec64_add(lhs: now, rhs: then);
252	igb_ptp_write_i210(adapter: igb, ts: (const struct timespec64 *)&now);
253
254	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
255
256	return 0;
257	}
258
259	static int igb_ptp_gettimex_82576(struct ptp_clock_info *ptp,
260	struct timespec64 *ts,
261	struct ptp_system_timestamp *sts)
262	{
263	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
264	ptp_caps);
265	struct e1000_hw *hw = &igb->hw;
266	unsigned long flags;
267	u32 lo, hi;
268	u64 ns;
269
270	spin_lock_irqsave(&igb->tmreg_lock, flags);
271
272	ptp_read_system_prets(sts);
273	lo = rd32(E1000_SYSTIML);
274	ptp_read_system_postts(sts);
275	hi = rd32(E1000_SYSTIMH);
276
277	ns = timecounter_cyc2time(tc: &igb->tc, cycle_tstamp: ((u64)hi << 32) | lo);
278
279	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
280
281	*ts = ns_to_timespec64(nsec: ns);
282
283	return 0;
284	}
285
286	static int igb_ptp_gettimex_82580(struct ptp_clock_info *ptp,
287	struct timespec64 *ts,
288	struct ptp_system_timestamp *sts)
289	{
290	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
291	ptp_caps);
292	struct e1000_hw *hw = &igb->hw;
293	unsigned long flags;
294	u32 lo, hi;
295	u64 ns;
296
297	spin_lock_irqsave(&igb->tmreg_lock, flags);
298
299	ptp_read_system_prets(sts);
300	rd32(E1000_SYSTIMR);
301	ptp_read_system_postts(sts);
302	lo = rd32(E1000_SYSTIML);
303	hi = rd32(E1000_SYSTIMH);
304
305	ns = timecounter_cyc2time(tc: &igb->tc, cycle_tstamp: ((u64)hi << 32) | lo);
306
307	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
308
309	*ts = ns_to_timespec64(nsec: ns);
310
311	return 0;
312	}
313
314	static int igb_ptp_gettimex_i210(struct ptp_clock_info *ptp,
315	struct timespec64 *ts,
316	struct ptp_system_timestamp *sts)
317	{
318	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
319	ptp_caps);
320	struct e1000_hw *hw = &igb->hw;
321	unsigned long flags;
322
323	spin_lock_irqsave(&igb->tmreg_lock, flags);
324
325	ptp_read_system_prets(sts);
326	rd32(E1000_SYSTIMR);
327	ptp_read_system_postts(sts);
328	ts->tv_nsec = rd32(E1000_SYSTIML);
329	ts->tv_sec = rd32(E1000_SYSTIMH);
330
331	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
332
333	return 0;
334	}
335
336	static int igb_ptp_settime_82576(struct ptp_clock_info *ptp,
337	const struct timespec64 *ts)
338	{
339	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
340	ptp_caps);
341	unsigned long flags;
342	u64 ns;
343
344	ns = timespec64_to_ns(ts);
345
346	spin_lock_irqsave(&igb->tmreg_lock, flags);
347
348	timecounter_init(tc: &igb->tc, cc: &igb->cc, start_tstamp: ns);
349
350	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
351
352	return 0;
353	}
354
355	static int igb_ptp_settime_i210(struct ptp_clock_info *ptp,
356	const struct timespec64 *ts)
357	{
358	struct igb_adapter *igb = container_of(ptp, struct igb_adapter,
359	ptp_caps);
360	unsigned long flags;
361
362	spin_lock_irqsave(&igb->tmreg_lock, flags);
363
364	igb_ptp_write_i210(adapter: igb, ts);
365
366	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
367
368	return 0;
369	}
370
371	static void igb_pin_direction(int pin, int input, u32 *ctrl, u32 *ctrl_ext)
372	{
373	u32 *ptr = pin < 2 ? ctrl : ctrl_ext;
374	static const u32 mask[IGB_N_SDP] = {
375	E1000_CTRL_SDP0_DIR,
376	E1000_CTRL_SDP1_DIR,
377	E1000_CTRL_EXT_SDP2_DIR,
378	E1000_CTRL_EXT_SDP3_DIR,
379	};
380
381	if (input)
382	*ptr &= ~mask[pin];
383	else
384	*ptr |= mask[pin];
385	}
386
387	static void igb_pin_extts(struct igb_adapter *igb, int chan, int pin)
388	{
389	static const u32 aux0_sel_sdp[IGB_N_SDP] = {
390	AUX0_SEL_SDP0, AUX0_SEL_SDP1, AUX0_SEL_SDP2, AUX0_SEL_SDP3,
391	};
392	static const u32 aux1_sel_sdp[IGB_N_SDP] = {
393	AUX1_SEL_SDP0, AUX1_SEL_SDP1, AUX1_SEL_SDP2, AUX1_SEL_SDP3,
394	};
395	static const u32 ts_sdp_en[IGB_N_SDP] = {
396	TS_SDP0_EN, TS_SDP1_EN, TS_SDP2_EN, TS_SDP3_EN,
397	};
398	struct e1000_hw *hw = &igb->hw;
399	u32 ctrl, ctrl_ext, tssdp = 0;
400
401	ctrl = rd32(E1000_CTRL);
402	ctrl_ext = rd32(E1000_CTRL_EXT);
403	tssdp = rd32(E1000_TSSDP);
404
405	igb_pin_direction(pin, input: 1, ctrl: &ctrl, ctrl_ext: &ctrl_ext);
406
407	/* Make sure this pin is not enabled as an output. */
408	tssdp &= ~ts_sdp_en[pin];
409
410	if (chan == 1) {
411	tssdp &= ~AUX1_SEL_SDP3;
412	tssdp |= aux1_sel_sdp[pin] | AUX1_TS_SDP_EN;
413	} else {
414	tssdp &= ~AUX0_SEL_SDP3;
415	tssdp |= aux0_sel_sdp[pin] | AUX0_TS_SDP_EN;
416	}
417
418	wr32(E1000_TSSDP, tssdp);
419	wr32(E1000_CTRL, ctrl);
420	wr32(E1000_CTRL_EXT, ctrl_ext);
421	}
422
423	static void igb_pin_perout(struct igb_adapter *igb, int chan, int pin, int freq)
424	{
425	static const u32 aux0_sel_sdp[IGB_N_SDP] = {
426	AUX0_SEL_SDP0, AUX0_SEL_SDP1, AUX0_SEL_SDP2, AUX0_SEL_SDP3,
427	};
428	static const u32 aux1_sel_sdp[IGB_N_SDP] = {
429	AUX1_SEL_SDP0, AUX1_SEL_SDP1, AUX1_SEL_SDP2, AUX1_SEL_SDP3,
430	};
431	static const u32 ts_sdp_en[IGB_N_SDP] = {
432	TS_SDP0_EN, TS_SDP1_EN, TS_SDP2_EN, TS_SDP3_EN,
433	};
434	static const u32 ts_sdp_sel_tt0[IGB_N_SDP] = {
435	TS_SDP0_SEL_TT0, TS_SDP1_SEL_TT0,
436	TS_SDP2_SEL_TT0, TS_SDP3_SEL_TT0,
437	};
438	static const u32 ts_sdp_sel_tt1[IGB_N_SDP] = {
439	TS_SDP0_SEL_TT1, TS_SDP1_SEL_TT1,
440	TS_SDP2_SEL_TT1, TS_SDP3_SEL_TT1,
441	};
442	static const u32 ts_sdp_sel_fc0[IGB_N_SDP] = {
443	TS_SDP0_SEL_FC0, TS_SDP1_SEL_FC0,
444	TS_SDP2_SEL_FC0, TS_SDP3_SEL_FC0,
445	};
446	static const u32 ts_sdp_sel_fc1[IGB_N_SDP] = {
447	TS_SDP0_SEL_FC1, TS_SDP1_SEL_FC1,
448	TS_SDP2_SEL_FC1, TS_SDP3_SEL_FC1,
449	};
450	static const u32 ts_sdp_sel_clr[IGB_N_SDP] = {
451	TS_SDP0_SEL_FC1, TS_SDP1_SEL_FC1,
452	TS_SDP2_SEL_FC1, TS_SDP3_SEL_FC1,
453	};
454	struct e1000_hw *hw = &igb->hw;
455	u32 ctrl, ctrl_ext, tssdp = 0;
456
457	ctrl = rd32(E1000_CTRL);
458	ctrl_ext = rd32(E1000_CTRL_EXT);
459	tssdp = rd32(E1000_TSSDP);
460
461	igb_pin_direction(pin, input: 0, ctrl: &ctrl, ctrl_ext: &ctrl_ext);
462
463	/* Make sure this pin is not enabled as an input. */
464	if ((tssdp & AUX0_SEL_SDP3) == aux0_sel_sdp[pin])
465	tssdp &= ~AUX0_TS_SDP_EN;
466
467	if ((tssdp & AUX1_SEL_SDP3) == aux1_sel_sdp[pin])
468	tssdp &= ~AUX1_TS_SDP_EN;
469
470	tssdp &= ~ts_sdp_sel_clr[pin];
471	if (freq) {
472	if (chan == 1)
473	tssdp |= ts_sdp_sel_fc1[pin];
474	else
475	tssdp |= ts_sdp_sel_fc0[pin];
476	} else {
477	if (chan == 1)
478	tssdp |= ts_sdp_sel_tt1[pin];
479	else
480	tssdp |= ts_sdp_sel_tt0[pin];
481	}
482	tssdp |= ts_sdp_en[pin];
483
484	wr32(E1000_TSSDP, tssdp);
485	wr32(E1000_CTRL, ctrl);
486	wr32(E1000_CTRL_EXT, ctrl_ext);
487	}
488
489	static int igb_ptp_feature_enable_82580(struct ptp_clock_info *ptp,
490	struct ptp_clock_request *rq, int on)
491	{
492	struct igb_adapter *igb =
493	container_of(ptp, struct igb_adapter, ptp_caps);
494	u32 tsauxc, tsim, tsauxc_mask, tsim_mask, trgttiml, trgttimh, systiml,
495	systimh, level_mask, level, rem;
496	struct e1000_hw *hw = &igb->hw;
497	struct timespec64 ts, start;
498	unsigned long flags;
499	u64 systim, now;
500	int pin = -1;
501	s64 ns;
502
503	switch (rq->type) {
504	case PTP_CLK_REQ_EXTTS:
505	/* Reject requests with unsupported flags */
506	if (rq->extts.flags & ~(PTP_ENABLE_FEATURE |
507	PTP_RISING_EDGE |
508	PTP_FALLING_EDGE |
509	PTP_STRICT_FLAGS))
510	return -EOPNOTSUPP;
511
512	if (on) {
513	pin = ptp_find_pin(ptp: igb->ptp_clock, func: PTP_PF_EXTTS,
514	chan: rq->extts.index);
515	if (pin < 0)
516	return -EBUSY;
517	}
518	if (rq->extts.index == 1) {
519	tsauxc_mask = TSAUXC_EN_TS1;
520	tsim_mask = TSINTR_AUTT1;
521	} else {
522	tsauxc_mask = TSAUXC_EN_TS0;
523	tsim_mask = TSINTR_AUTT0;
524	}
525	spin_lock_irqsave(&igb->tmreg_lock, flags);
526	tsauxc = rd32(E1000_TSAUXC);
527	tsim = rd32(E1000_TSIM);
528	if (on) {
529	igb_pin_extts(igb, chan: rq->extts.index, pin);
530	tsauxc |= tsauxc_mask;
531	tsim |= tsim_mask;
532	} else {
533	tsauxc &= ~tsauxc_mask;
534	tsim &= ~tsim_mask;
535	}
536	wr32(E1000_TSAUXC, tsauxc);
537	wr32(E1000_TSIM, tsim);
538	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
539	return 0;
540
541	case PTP_CLK_REQ_PEROUT:
542	/* Reject requests with unsupported flags */
543	if (rq->perout.flags)
544	return -EOPNOTSUPP;
545
546	if (on) {
547	pin = ptp_find_pin(ptp: igb->ptp_clock, func: PTP_PF_PEROUT,
548	chan: rq->perout.index);
549	if (pin < 0)
550	return -EBUSY;
551	}
552	ts.tv_sec = rq->perout.period.sec;
553	ts.tv_nsec = rq->perout.period.nsec;
554	ns = timespec64_to_ns(ts: &ts);
555	ns = ns >> 1;
556	if (on && ns < 8LL)
557	return -EINVAL;
558	ts = ns_to_timespec64(nsec: ns);
559	if (rq->perout.index == 1) {
560	tsauxc_mask = TSAUXC_EN_TT1;
561	tsim_mask = TSINTR_TT1;
562	trgttiml = E1000_TRGTTIML1;
563	trgttimh = E1000_TRGTTIMH1;
564	} else {
565	tsauxc_mask = TSAUXC_EN_TT0;
566	tsim_mask = TSINTR_TT0;
567	trgttiml = E1000_TRGTTIML0;
568	trgttimh = E1000_TRGTTIMH0;
569	}
570	spin_lock_irqsave(&igb->tmreg_lock, flags);
571	tsauxc = rd32(E1000_TSAUXC);
572	tsim = rd32(E1000_TSIM);
573	if (rq->perout.index == 1) {
574	tsauxc &= ~(TSAUXC_EN_TT1 | TSAUXC_EN_CLK1 | TSAUXC_ST1);
575	tsim &= ~TSINTR_TT1;
576	} else {
577	tsauxc &= ~(TSAUXC_EN_TT0 | TSAUXC_EN_CLK0 | TSAUXC_ST0);
578	tsim &= ~TSINTR_TT0;
579	}
580	if (on) {
581	int i = rq->perout.index;
582
583	/* read systim registers in sequence */
584	rd32(E1000_SYSTIMR);
585	systiml = rd32(E1000_SYSTIML);
586	systimh = rd32(E1000_SYSTIMH);
587	systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml);
588	now = timecounter_cyc2time(tc: &igb->tc, cycle_tstamp: systim);
589
590	if (pin < 2) {
591	level_mask = (i == 1) ? 0x80000 : 0x40000;
592	level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0;
593	} else {
594	level_mask = (i == 1) ? 0x80 : 0x40;
595	level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0;
596	}
597
598	div_u64_rem(dividend: now, divisor: ns, remainder: &rem);
599	systim = systim + (ns - rem);
600
601	/* synchronize pin level with rising/falling edges */
602	div_u64_rem(dividend: now, divisor: ns << 1, remainder: &rem);
603	if (rem < ns) {
604	/* first half of period */
605	if (level == 0) {
606	/* output is already low, skip this period */
607	systim += ns;
608	}
609	} else {
610	/* second half of period */
611	if (level == 1) {
612	/* output is already high, skip this period */
613	systim += ns;
614	}
615	}
616
617	start = ns_to_timespec64(nsec: systim + (ns - rem));
618	igb_pin_perout(igb, chan: i, pin, freq: 0);
619	igb->perout[i].start.tv_sec = start.tv_sec;
620	igb->perout[i].start.tv_nsec = start.tv_nsec;
621	igb->perout[i].period.tv_sec = ts.tv_sec;
622	igb->perout[i].period.tv_nsec = ts.tv_nsec;
623
624	wr32(trgttiml, (u32)systim);
625	wr32(trgttimh, ((u32)(systim >> 32)) & 0xFF);
626	tsauxc |= tsauxc_mask;
627	tsim |= tsim_mask;
628	}
629	wr32(E1000_TSAUXC, tsauxc);
630	wr32(E1000_TSIM, tsim);
631	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
632	return 0;
633
634	case PTP_CLK_REQ_PPS:
635	return -EOPNOTSUPP;
636	}
637
638	return -EOPNOTSUPP;
639	}
640
641	static int igb_ptp_feature_enable_i210(struct ptp_clock_info *ptp,
642	struct ptp_clock_request *rq, int on)
643	{
644	struct igb_adapter *igb =
645	container_of(ptp, struct igb_adapter, ptp_caps);
646	struct e1000_hw *hw = &igb->hw;
647	u32 tsauxc, tsim, tsauxc_mask, tsim_mask, trgttiml, trgttimh, freqout;
648	unsigned long flags;
649	struct timespec64 ts;
650	int use_freq = 0, pin = -1;
651	s64 ns;
652
653	switch (rq->type) {
654	case PTP_CLK_REQ_EXTTS:
655	/* Reject requests with unsupported flags */
656	if (rq->extts.flags & ~(PTP_ENABLE_FEATURE |
657	PTP_RISING_EDGE |
658	PTP_FALLING_EDGE |
659	PTP_STRICT_FLAGS))
660	return -EOPNOTSUPP;
661
662	/* Reject requests failing to enable both edges. */
663	if ((rq->extts.flags & PTP_STRICT_FLAGS) &&
664	(rq->extts.flags & PTP_ENABLE_FEATURE) &&
665	(rq->extts.flags & PTP_EXTTS_EDGES) != PTP_EXTTS_EDGES)
666	return -EOPNOTSUPP;
667
668	if (on) {
669	pin = ptp_find_pin(ptp: igb->ptp_clock, func: PTP_PF_EXTTS,
670	chan: rq->extts.index);
671	if (pin < 0)
672	return -EBUSY;
673	}
674	if (rq->extts.index == 1) {
675	tsauxc_mask = TSAUXC_EN_TS1;
676	tsim_mask = TSINTR_AUTT1;
677	} else {
678	tsauxc_mask = TSAUXC_EN_TS0;
679	tsim_mask = TSINTR_AUTT0;
680	}
681	spin_lock_irqsave(&igb->tmreg_lock, flags);
682	tsauxc = rd32(E1000_TSAUXC);
683	tsim = rd32(E1000_TSIM);
684	if (on) {
685	igb_pin_extts(igb, chan: rq->extts.index, pin);
686	tsauxc |= tsauxc_mask;
687	tsim |= tsim_mask;
688	} else {
689	tsauxc &= ~tsauxc_mask;
690	tsim &= ~tsim_mask;
691	}
692	wr32(E1000_TSAUXC, tsauxc);
693	wr32(E1000_TSIM, tsim);
694	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
695	return 0;
696
697	case PTP_CLK_REQ_PEROUT:
698	/* Reject requests with unsupported flags */
699	if (rq->perout.flags)
700	return -EOPNOTSUPP;
701
702	if (on) {
703	pin = ptp_find_pin(ptp: igb->ptp_clock, func: PTP_PF_PEROUT,
704	chan: rq->perout.index);
705	if (pin < 0)
706	return -EBUSY;
707	}
708	ts.tv_sec = rq->perout.period.sec;
709	ts.tv_nsec = rq->perout.period.nsec;
710	ns = timespec64_to_ns(ts: &ts);
711	ns = ns >> 1;
712	if (on && ((ns <= 70000000LL) || (ns == 125000000LL) ||
713	(ns == 250000000LL) || (ns == 500000000LL))) {
714	if (ns < 8LL)
715	return -EINVAL;
716	use_freq = 1;
717	}
718	ts = ns_to_timespec64(nsec: ns);
719	if (rq->perout.index == 1) {
720	if (use_freq) {
721	tsauxc_mask = TSAUXC_EN_CLK1 | TSAUXC_ST1;
722	tsim_mask = 0;
723	} else {
724	tsauxc_mask = TSAUXC_EN_TT1;
725	tsim_mask = TSINTR_TT1;
726	}
727	trgttiml = E1000_TRGTTIML1;
728	trgttimh = E1000_TRGTTIMH1;
729	freqout = E1000_FREQOUT1;
730	} else {
731	if (use_freq) {
732	tsauxc_mask = TSAUXC_EN_CLK0 | TSAUXC_ST0;
733	tsim_mask = 0;
734	} else {
735	tsauxc_mask = TSAUXC_EN_TT0;
736	tsim_mask = TSINTR_TT0;
737	}
738	trgttiml = E1000_TRGTTIML0;
739	trgttimh = E1000_TRGTTIMH0;
740	freqout = E1000_FREQOUT0;
741	}
742	spin_lock_irqsave(&igb->tmreg_lock, flags);
743	tsauxc = rd32(E1000_TSAUXC);
744	tsim = rd32(E1000_TSIM);
745	if (rq->perout.index == 1) {
746	tsauxc &= ~(TSAUXC_EN_TT1 | TSAUXC_EN_CLK1 | TSAUXC_ST1);
747	tsim &= ~TSINTR_TT1;
748	} else {
749	tsauxc &= ~(TSAUXC_EN_TT0 | TSAUXC_EN_CLK0 | TSAUXC_ST0);
750	tsim &= ~TSINTR_TT0;
751	}
752	if (on) {
753	int i = rq->perout.index;
754	igb_pin_perout(igb, chan: i, pin, freq: use_freq);
755	igb->perout[i].start.tv_sec = rq->perout.start.sec;
756	igb->perout[i].start.tv_nsec = rq->perout.start.nsec;
757	igb->perout[i].period.tv_sec = ts.tv_sec;
758	igb->perout[i].period.tv_nsec = ts.tv_nsec;
759	wr32(trgttimh, rq->perout.start.sec);
760	wr32(trgttiml, rq->perout.start.nsec);
761	if (use_freq)
762	wr32(freqout, ns);
763	tsauxc |= tsauxc_mask;
764	tsim |= tsim_mask;
765	}
766	wr32(E1000_TSAUXC, tsauxc);
767	wr32(E1000_TSIM, tsim);
768	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
769	return 0;
770
771	case PTP_CLK_REQ_PPS:
772	spin_lock_irqsave(&igb->tmreg_lock, flags);
773	tsim = rd32(E1000_TSIM);
774	if (on)
775	tsim |= TSINTR_SYS_WRAP;
776	else
777	tsim &= ~TSINTR_SYS_WRAP;
778	igb->pps_sys_wrap_on = !!on;
779	wr32(E1000_TSIM, tsim);
780	spin_unlock_irqrestore(lock: &igb->tmreg_lock, flags);
781	return 0;
782	}
783
784	return -EOPNOTSUPP;
785	}
786
787	static int igb_ptp_feature_enable(struct ptp_clock_info *ptp,
788	struct ptp_clock_request *rq, int on)
789	{
790	return -EOPNOTSUPP;
791	}
792
793	static int igb_ptp_verify_pin(struct ptp_clock_info *ptp, unsigned int pin,
794	enum ptp_pin_function func, unsigned int chan)
795	{
796	switch (func) {
797	case PTP_PF_NONE:
798	case PTP_PF_EXTTS:
799	case PTP_PF_PEROUT:
800	break;
801	case PTP_PF_PHYSYNC:
802	return -1;
803	}
804	return 0;
805	}
806
807	/**
808	* igb_ptp_tx_work
809	* @work: pointer to work struct
810	*
811	* This work function polls the TSYNCTXCTL valid bit to determine when a
812	* timestamp has been taken for the current stored skb.
813	**/
814	static void igb_ptp_tx_work(struct work_struct *work)
815	{
816	struct igb_adapter *adapter = container_of(work, struct igb_adapter,
817	ptp_tx_work);
818	struct e1000_hw *hw = &adapter->hw;
819	u32 tsynctxctl;
820
821	if (!adapter->ptp_tx_skb)
822	return;
823
824	if (time_is_before_jiffies(adapter->ptp_tx_start +
825	IGB_PTP_TX_TIMEOUT)) {
826	dev_kfree_skb_any(skb: adapter->ptp_tx_skb);
827	adapter->ptp_tx_skb = NULL;
828	clear_bit_unlock(nr: __IGB_PTP_TX_IN_PROGRESS, addr: &adapter->state);
829	adapter->tx_hwtstamp_timeouts++;
830	/* Clear the tx valid bit in TSYNCTXCTL register to enable
831	* interrupt
832	*/
833	rd32(E1000_TXSTMPH);
834	dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
835	return;
836	}
837
838	tsynctxctl = rd32(E1000_TSYNCTXCTL);
839	if (tsynctxctl & E1000_TSYNCTXCTL_VALID)
840	igb_ptp_tx_hwtstamp(adapter);
841	else
842	/* reschedule to check later */
843	schedule_work(work: &adapter->ptp_tx_work);
844	}
845
846	static void igb_ptp_overflow_check(struct work_struct *work)
847	{
848	struct igb_adapter *igb =
849	container_of(work, struct igb_adapter, ptp_overflow_work.work);
850	struct timespec64 ts;
851	u64 ns;
852
853	/* Update the timecounter */
854	ns = timecounter_read(tc: &igb->tc);
855
856	ts = ns_to_timespec64(nsec: ns);
857	pr_debug("igb overflow check at %lld.%09lu\n",
858	(long long) ts.tv_sec, ts.tv_nsec);
859
860	schedule_delayed_work(dwork: &igb->ptp_overflow_work,
861	IGB_SYSTIM_OVERFLOW_PERIOD);
862	}
863
864	/**
865	* igb_ptp_rx_hang - detect error case when Rx timestamp registers latched
866	* @adapter: private network adapter structure
867	*
868	* This watchdog task is scheduled to detect error case where hardware has
869	* dropped an Rx packet that was timestamped when the ring is full. The
870	* particular error is rare but leaves the device in a state unable to timestamp
871	* any future packets.
872	**/
873	void igb_ptp_rx_hang(struct igb_adapter *adapter)
874	{
875	struct e1000_hw *hw = &adapter->hw;
876	u32 tsyncrxctl = rd32(E1000_TSYNCRXCTL);
877	unsigned long rx_event;
878
879	/* Other hardware uses per-packet timestamps */
880	if (hw->mac.type != e1000_82576)
881	return;
882
883	/* If we don't have a valid timestamp in the registers, just update the
884	* timeout counter and exit
885	*/
886	if (!(tsyncrxctl & E1000_TSYNCRXCTL_VALID)) {
887	adapter->last_rx_ptp_check = jiffies;
888	return;
889	}
890
891	/* Determine the most recent watchdog or rx_timestamp event */
892	rx_event = adapter->last_rx_ptp_check;
893	if (time_after(adapter->last_rx_timestamp, rx_event))
894	rx_event = adapter->last_rx_timestamp;
895
896	/* Only need to read the high RXSTMP register to clear the lock */
897	if (time_is_before_jiffies(rx_event + 5 * HZ)) {
898	rd32(E1000_RXSTMPH);
899	adapter->last_rx_ptp_check = jiffies;
900	adapter->rx_hwtstamp_cleared++;
901	dev_warn(&adapter->pdev->dev, "clearing Rx timestamp hang\n");
902	}
903	}
904
905	/**
906	* igb_ptp_tx_hang - detect error case where Tx timestamp never finishes
907	* @adapter: private network adapter structure
908	*/
909	void igb_ptp_tx_hang(struct igb_adapter *adapter)
910	{
911	struct e1000_hw *hw = &adapter->hw;
912	bool timeout = time_is_before_jiffies(adapter->ptp_tx_start +
913	IGB_PTP_TX_TIMEOUT);
914
915	if (!adapter->ptp_tx_skb)
916	return;
917
918	if (!test_bit(__IGB_PTP_TX_IN_PROGRESS, &adapter->state))
919	return;
920
921	/* If we haven't received a timestamp within the timeout, it is
922	* reasonable to assume that it will never occur, so we can unlock the
923	* timestamp bit when this occurs.
924	*/
925	if (timeout) {
926	cancel_work_sync(work: &adapter->ptp_tx_work);
927	dev_kfree_skb_any(skb: adapter->ptp_tx_skb);
928	adapter->ptp_tx_skb = NULL;
929	clear_bit_unlock(nr: __IGB_PTP_TX_IN_PROGRESS, addr: &adapter->state);
930	adapter->tx_hwtstamp_timeouts++;
931	/* Clear the tx valid bit in TSYNCTXCTL register to enable
932	* interrupt
933	*/
934	rd32(E1000_TXSTMPH);
935	dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n");
936	}
937	}
938
939	/**
940	* igb_ptp_tx_hwtstamp - utility function which checks for TX time stamp
941	* @adapter: Board private structure.
942	*
943	* If we were asked to do hardware stamping and such a time stamp is
944	* available, then it must have been for this skb here because we only
945	* allow only one such packet into the queue.
946	**/
947	static void igb_ptp_tx_hwtstamp(struct igb_adapter *adapter)
948	{
949	struct sk_buff *skb = adapter->ptp_tx_skb;
950	struct e1000_hw *hw = &adapter->hw;
951	struct skb_shared_hwtstamps shhwtstamps;
952	u64 regval;
953	int adjust = 0;
954
955	regval = rd32(E1000_TXSTMPL);
956	regval |= (u64)rd32(E1000_TXSTMPH) << 32;
957
958	igb_ptp_systim_to_hwtstamp(adapter, hwtstamps: &shhwtstamps, systim: regval);
959	/* adjust timestamp for the TX latency based on link speed */
960	if (adapter->hw.mac.type == e1000_i210) {
961	switch (adapter->link_speed) {
962	case SPEED_10:
963	adjust = IGB_I210_TX_LATENCY_10;
964	break;
965	case SPEED_100:
966	adjust = IGB_I210_TX_LATENCY_100;
967	break;
968	case SPEED_1000:
969	adjust = IGB_I210_TX_LATENCY_1000;
970	break;
971	}
972	}
973
974	shhwtstamps.hwtstamp =
975	ktime_add_ns(shhwtstamps.hwtstamp, adjust);
976
977	/* Clear the lock early before calling skb_tstamp_tx so that
978	* applications are not woken up before the lock bit is clear. We use
979	* a copy of the skb pointer to ensure other threads can't change it
980	* while we're notifying the stack.
981	*/
982	adapter->ptp_tx_skb = NULL;
983	clear_bit_unlock(nr: __IGB_PTP_TX_IN_PROGRESS, addr: &adapter->state);
984
985	/* Notify the stack and free the skb after we've unlocked */
986	skb_tstamp_tx(orig_skb: skb, hwtstamps: &shhwtstamps);
987	dev_kfree_skb_any(skb);
988	}
989
990	/**
991	* igb_ptp_rx_pktstamp - retrieve Rx per packet timestamp
992	* @q_vector: Pointer to interrupt specific structure
993	* @va: Pointer to address containing Rx buffer
994	* @timestamp: Pointer where timestamp will be stored
995	*
996	* This function is meant to retrieve a timestamp from the first buffer of an
997	* incoming frame. The value is stored in little endian format starting on
998	* byte 8
999	*
1000	* Returns: The timestamp header length or 0 if not available
1001	**/
1002	int igb_ptp_rx_pktstamp(struct igb_q_vector *q_vector, void *va,
1003	ktime_t *timestamp)
1004	{
1005	struct igb_adapter *adapter = q_vector->adapter;
1006	struct skb_shared_hwtstamps ts;
1007	__le64 *regval = (__le64 *)va;
1008	int adjust = 0;
1009
1010	if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1011	return 0;
1012
1013	/* The timestamp is recorded in little endian format.
1014	* DWORD: 0 1 2 3
1015	* Field: Reserved Reserved SYSTIML SYSTIMH
1016	*/
1017
1018	/* check reserved dwords are zero, be/le doesn't matter for zero */
1019	if (regval[0])
1020	return 0;
1021
1022	igb_ptp_systim_to_hwtstamp(adapter, hwtstamps: &ts, le64_to_cpu(regval[1]));
1023
1024	/* adjust timestamp for the RX latency based on link speed */
1025	if (adapter->hw.mac.type == e1000_i210) {
1026	switch (adapter->link_speed) {
1027	case SPEED_10:
1028	adjust = IGB_I210_RX_LATENCY_10;
1029	break;
1030	case SPEED_100:
1031	adjust = IGB_I210_RX_LATENCY_100;
1032	break;
1033	case SPEED_1000:
1034	adjust = IGB_I210_RX_LATENCY_1000;
1035	break;
1036	}
1037	}
1038
1039	*timestamp = ktime_sub_ns(ts.hwtstamp, adjust);
1040
1041	return IGB_TS_HDR_LEN;
1042	}
1043
1044	/**
1045	* igb_ptp_rx_rgtstamp - retrieve Rx timestamp stored in register
1046	* @q_vector: Pointer to interrupt specific structure
1047	* @skb: Buffer containing timestamp and packet
1048	*
1049	* This function is meant to retrieve a timestamp from the internal registers
1050	* of the adapter and store it in the skb.
1051	**/
1052	void igb_ptp_rx_rgtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
1053	{
1054	struct igb_adapter *adapter = q_vector->adapter;
1055	struct e1000_hw *hw = &adapter->hw;
1056	int adjust = 0;
1057	u64 regval;
1058
1059	if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1060	return;
1061
1062	/* If this bit is set, then the RX registers contain the time stamp. No
1063	* other packet will be time stamped until we read these registers, so
1064	* read the registers to make them available again. Because only one
1065	* packet can be time stamped at a time, we know that the register
1066	* values must belong to this one here and therefore we don't need to
1067	* compare any of the additional attributes stored for it.
1068	*
1069	* If nothing went wrong, then it should have a shared tx_flags that we
1070	* can turn into a skb_shared_hwtstamps.
1071	*/
1072	if (!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
1073	return;
1074
1075	regval = rd32(E1000_RXSTMPL);
1076	regval |= (u64)rd32(E1000_RXSTMPH) << 32;
1077
1078	igb_ptp_systim_to_hwtstamp(adapter, hwtstamps: skb_hwtstamps(skb), systim: regval);
1079
1080	/* adjust timestamp for the RX latency based on link speed */
1081	if (adapter->hw.mac.type == e1000_i210) {
1082	switch (adapter->link_speed) {
1083	case SPEED_10:
1084	adjust = IGB_I210_RX_LATENCY_10;
1085	break;
1086	case SPEED_100:
1087	adjust = IGB_I210_RX_LATENCY_100;
1088	break;
1089	case SPEED_1000:
1090	adjust = IGB_I210_RX_LATENCY_1000;
1091	break;
1092	}
1093	}
1094	skb_hwtstamps(skb)->hwtstamp =
1095	ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust);
1096
1097	/* Update the last_rx_timestamp timer in order to enable watchdog check
1098	* for error case of latched timestamp on a dropped packet.
1099	*/
1100	adapter->last_rx_timestamp = jiffies;
1101	}
1102
1103	/**
1104	* igb_ptp_get_ts_config - get hardware time stamping config
1105	* @netdev: netdev struct
1106	* @ifr: interface struct
1107	*
1108	* Get the hwtstamp_config settings to return to the user. Rather than attempt
1109	* to deconstruct the settings from the registers, just return a shadow copy
1110	* of the last known settings.
1111	**/
1112	int igb_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr)
1113	{
1114	struct igb_adapter *adapter = netdev_priv(dev: netdev);
1115	struct hwtstamp_config *config = &adapter->tstamp_config;
1116
1117	return copy_to_user(to: ifr->ifr_data, from: config, n: sizeof(*config)) ?
1118	-EFAULT : 0;
1119	}
1120
1121	/**
1122	* igb_ptp_set_timestamp_mode - setup hardware for timestamping
1123	* @adapter: networking device structure
1124	* @config: hwtstamp configuration
1125	*
1126	* Outgoing time stamping can be enabled and disabled. Play nice and
1127	* disable it when requested, although it shouldn't case any overhead
1128	* when no packet needs it. At most one packet in the queue may be
1129	* marked for time stamping, otherwise it would be impossible to tell
1130	* for sure to which packet the hardware time stamp belongs.
1131	*
1132	* Incoming time stamping has to be configured via the hardware
1133	* filters. Not all combinations are supported, in particular event
1134	* type has to be specified. Matching the kind of event packet is
1135	* not supported, with the exception of "all V2 events regardless of
1136	* level 2 or 4".
1137	*/
1138	static int igb_ptp_set_timestamp_mode(struct igb_adapter *adapter,
1139	struct hwtstamp_config *config)
1140	{
1141	struct e1000_hw *hw = &adapter->hw;
1142	u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
1143	u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
1144	u32 tsync_rx_cfg = 0;
1145	bool is_l4 = false;
1146	bool is_l2 = false;
1147	u32 regval;
1148
1149	switch (config->tx_type) {
1150	case HWTSTAMP_TX_OFF:
1151	tsync_tx_ctl = 0;
1152	break;
1153	case HWTSTAMP_TX_ON:
1154	break;
1155	default:
1156	return -ERANGE;
1157	}
1158
1159	switch (config->rx_filter) {
1160	case HWTSTAMP_FILTER_NONE:
1161	tsync_rx_ctl = 0;
1162	break;
1163	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1164	tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
1165	tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
1166	is_l4 = true;
1167	break;
1168	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1169	tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
1170	tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
1171	is_l4 = true;
1172	break;
1173	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1174	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1175	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1176	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1177	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1178	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1179	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1180	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1181	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1182	tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
1183	config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
1184	is_l2 = true;
1185	is_l4 = true;
1186	break;
1187	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1188	case HWTSTAMP_FILTER_NTP_ALL:
1189	case HWTSTAMP_FILTER_ALL:
1190	/* 82576 cannot timestamp all packets, which it needs to do to
1191	* support both V1 Sync and Delay_Req messages
1192	*/
1193	if (hw->mac.type != e1000_82576) {
1194	tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
1195	config->rx_filter = HWTSTAMP_FILTER_ALL;
1196	break;
1197	}
1198	fallthrough;
1199	default:
1200	config->rx_filter = HWTSTAMP_FILTER_NONE;
1201	return -ERANGE;
1202	}
1203
1204	if (hw->mac.type == e1000_82575) {
1205	if (tsync_rx_ctl | tsync_tx_ctl)
1206	return -EINVAL;
1207	return 0;
1208	}
1209
1210	/* Per-packet timestamping only works if all packets are
1211	* timestamped, so enable timestamping in all packets as
1212	* long as one Rx filter was configured.
1213	*/
1214	if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
1215	tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
1216	tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
1217	config->rx_filter = HWTSTAMP_FILTER_ALL;
1218	is_l2 = true;
1219	is_l4 = true;
1220
1221	if ((hw->mac.type == e1000_i210) ||
1222	(hw->mac.type == e1000_i211)) {
1223	regval = rd32(E1000_RXPBS);
1224	regval |= E1000_RXPBS_CFG_TS_EN;
1225	wr32(E1000_RXPBS, regval);
1226	}
1227	}
1228
1229	/* enable/disable TX */
1230	regval = rd32(E1000_TSYNCTXCTL);
1231	regval &= ~E1000_TSYNCTXCTL_ENABLED;
1232	regval |= tsync_tx_ctl;
1233	wr32(E1000_TSYNCTXCTL, regval);
1234
1235	/* enable/disable RX */
1236	regval = rd32(E1000_TSYNCRXCTL);
1237	regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
1238	regval |= tsync_rx_ctl;
1239	wr32(E1000_TSYNCRXCTL, regval);
1240
1241	/* define which PTP packets are time stamped */
1242	wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
1243
1244	/* define ethertype filter for timestamped packets */
1245	if (is_l2)
1246	wr32(E1000_ETQF(IGB_ETQF_FILTER_1588),
1247	(E1000_ETQF_FILTER_ENABLE | /* enable filter */
1248	E1000_ETQF_1588 | /* enable timestamping */
1249	ETH_P_1588)); /* 1588 eth protocol type */
1250	else
1251	wr32(E1000_ETQF(IGB_ETQF_FILTER_1588), 0);
1252
1253	/* L4 Queue Filter[3]: filter by destination port and protocol */
1254	if (is_l4) {
1255	u32 ftqf = (IPPROTO_UDP /* UDP */
1256	| E1000_FTQF_VF_BP /* VF not compared */
1257	| E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
1258	| E1000_FTQF_MASK); /* mask all inputs */
1259	ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
1260
1261	wr32(E1000_IMIR(3), (__force unsigned int)htons(PTP_EV_PORT));
1262	wr32(E1000_IMIREXT(3),
1263	(E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
1264	if (hw->mac.type == e1000_82576) {
1265	/* enable source port check */
1266	wr32(E1000_SPQF(3), (__force unsigned int)htons(PTP_EV_PORT));
1267	ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
1268	}
1269	wr32(E1000_FTQF(3), ftqf);
1270	} else {
1271	wr32(E1000_FTQF(3), E1000_FTQF_MASK);
1272	}
1273	wrfl();
1274
1275	/* clear TX/RX time stamp registers, just to be sure */
1276	regval = rd32(E1000_TXSTMPL);
1277	regval = rd32(E1000_TXSTMPH);
1278	regval = rd32(E1000_RXSTMPL);
1279	regval = rd32(E1000_RXSTMPH);
1280
1281	return 0;
1282	}
1283
1284	/**
1285	* igb_ptp_set_ts_config - set hardware time stamping config
1286	* @netdev: netdev struct
1287	* @ifr: interface struct
1288	*
1289	**/
1290	int igb_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr)
1291	{
1292	struct igb_adapter *adapter = netdev_priv(dev: netdev);
1293	struct hwtstamp_config config;
1294	int err;
1295
1296	if (copy_from_user(to: &config, from: ifr->ifr_data, n: sizeof(config)))
1297	return -EFAULT;
1298
1299	err = igb_ptp_set_timestamp_mode(adapter, config: &config);
1300	if (err)
1301	return err;
1302
1303	/* save these settings for future reference */
1304	memcpy(&adapter->tstamp_config, &config,
1305	sizeof(adapter->tstamp_config));
1306
1307	return copy_to_user(to: ifr->ifr_data, from: &config, n: sizeof(config)) ?
1308	-EFAULT : 0;
1309	}
1310
1311	/**
1312	* igb_ptp_init - Initialize PTP functionality
1313	* @adapter: Board private structure
1314	*
1315	* This function is called at device probe to initialize the PTP
1316	* functionality.
1317	*/
1318	void igb_ptp_init(struct igb_adapter *adapter)
1319	{
1320	struct e1000_hw *hw = &adapter->hw;
1321	struct net_device *netdev = adapter->netdev;
1322
1323	switch (hw->mac.type) {
1324	case e1000_82576:
1325	snprintf(buf: adapter->ptp_caps.name, size: 16, fmt: "%pm", netdev->dev_addr);
1326	adapter->ptp_caps.owner = THIS_MODULE;
1327	adapter->ptp_caps.max_adj = 999999881;
1328	adapter->ptp_caps.n_ext_ts = 0;
1329	adapter->ptp_caps.pps = 0;
1330	adapter->ptp_caps.adjfine = igb_ptp_adjfine_82576;
1331	adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576;
1332	adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_82576;
1333	adapter->ptp_caps.settime64 = igb_ptp_settime_82576;
1334	adapter->ptp_caps.enable = igb_ptp_feature_enable;
1335	adapter->cc.read = igb_ptp_read_82576;
1336	adapter->cc.mask = CYCLECOUNTER_MASK(64);
1337	adapter->cc.mult = 1;
1338	adapter->cc.shift = IGB_82576_TSYNC_SHIFT;
1339	adapter->ptp_flags |= IGB_PTP_OVERFLOW_CHECK;
1340	break;
1341	case e1000_82580:
1342	case e1000_i354:
1343	case e1000_i350:
1344	igb_ptp_sdp_init(adapter);
1345	snprintf(buf: adapter->ptp_caps.name, size: 16, fmt: "%pm", netdev->dev_addr);
1346	adapter->ptp_caps.owner = THIS_MODULE;
1347	adapter->ptp_caps.max_adj = 62499999;
1348	adapter->ptp_caps.n_ext_ts = IGB_N_EXTTS;
1349	adapter->ptp_caps.n_per_out = IGB_N_PEROUT;
1350	adapter->ptp_caps.n_pins = IGB_N_SDP;
1351	adapter->ptp_caps.pps = 0;
1352	adapter->ptp_caps.pin_config = adapter->sdp_config;
1353	adapter->ptp_caps.adjfine = igb_ptp_adjfine_82580;
1354	adapter->ptp_caps.adjtime = igb_ptp_adjtime_82576;
1355	adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_82580;
1356	adapter->ptp_caps.settime64 = igb_ptp_settime_82576;
1357	adapter->ptp_caps.enable = igb_ptp_feature_enable_82580;
1358	adapter->ptp_caps.verify = igb_ptp_verify_pin;
1359	adapter->cc.read = igb_ptp_read_82580;
1360	adapter->cc.mask = CYCLECOUNTER_MASK(IGB_NBITS_82580);
1361	adapter->cc.mult = 1;
1362	adapter->cc.shift = 0;
1363	adapter->ptp_flags |= IGB_PTP_OVERFLOW_CHECK;
1364	break;
1365	case e1000_i210:
1366	case e1000_i211:
1367	igb_ptp_sdp_init(adapter);
1368	snprintf(buf: adapter->ptp_caps.name, size: 16, fmt: "%pm", netdev->dev_addr);
1369	adapter->ptp_caps.owner = THIS_MODULE;
1370	adapter->ptp_caps.max_adj = 62499999;
1371	adapter->ptp_caps.n_ext_ts = IGB_N_EXTTS;
1372	adapter->ptp_caps.n_per_out = IGB_N_PEROUT;
1373	adapter->ptp_caps.n_pins = IGB_N_SDP;
1374	adapter->ptp_caps.pps = 1;
1375	adapter->ptp_caps.pin_config = adapter->sdp_config;
1376	adapter->ptp_caps.adjfine = igb_ptp_adjfine_82580;
1377	adapter->ptp_caps.adjtime = igb_ptp_adjtime_i210;
1378	adapter->ptp_caps.gettimex64 = igb_ptp_gettimex_i210;
1379	adapter->ptp_caps.settime64 = igb_ptp_settime_i210;
1380	adapter->ptp_caps.enable = igb_ptp_feature_enable_i210;
1381	adapter->ptp_caps.verify = igb_ptp_verify_pin;
1382	break;
1383	default:
1384	adapter->ptp_clock = NULL;
1385	return;
1386	}
1387
1388	adapter->ptp_clock = ptp_clock_register(info: &adapter->ptp_caps,
1389	parent: &adapter->pdev->dev);
1390	if (IS_ERR(ptr: adapter->ptp_clock)) {
1391	adapter->ptp_clock = NULL;
1392	dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n");
1393	} else if (adapter->ptp_clock) {
1394	dev_info(&adapter->pdev->dev, "added PHC on %s\n",
1395	adapter->netdev->name);
1396	adapter->ptp_flags |= IGB_PTP_ENABLED;
1397
1398	spin_lock_init(&adapter->tmreg_lock);
1399	INIT_WORK(&adapter->ptp_tx_work, igb_ptp_tx_work);
1400
1401	if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1402	INIT_DELAYED_WORK(&adapter->ptp_overflow_work,
1403	igb_ptp_overflow_check);
1404
1405	adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
1406	adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF;
1407
1408	igb_ptp_reset(adapter);
1409	}
1410	}
1411
1412	/**
1413	* igb_ptp_sdp_init - utility function which inits the SDP config structs
1414	* @adapter: Board private structure.
1415	**/
1416	void igb_ptp_sdp_init(struct igb_adapter *adapter)
1417	{
1418	int i;
1419
1420	for (i = 0; i < IGB_N_SDP; i++) {
1421	struct ptp_pin_desc *ppd = &adapter->sdp_config[i];
1422
1423	snprintf(buf: ppd->name, size: sizeof(ppd->name), fmt: "SDP%d", i);
1424	ppd->index = i;
1425	ppd->func = PTP_PF_NONE;
1426	}
1427	}
1428
1429	/**
1430	* igb_ptp_suspend - Disable PTP work items and prepare for suspend
1431	* @adapter: Board private structure
1432	*
1433	* This function stops the overflow check work and PTP Tx timestamp work, and
1434	* will prepare the device for OS suspend.
1435	*/
1436	void igb_ptp_suspend(struct igb_adapter *adapter)
1437	{
1438	if (!(adapter->ptp_flags & IGB_PTP_ENABLED))
1439	return;
1440
1441	if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1442	cancel_delayed_work_sync(dwork: &adapter->ptp_overflow_work);
1443
1444	cancel_work_sync(work: &adapter->ptp_tx_work);
1445	if (adapter->ptp_tx_skb) {
1446	dev_kfree_skb_any(skb: adapter->ptp_tx_skb);
1447	adapter->ptp_tx_skb = NULL;
1448	clear_bit_unlock(nr: __IGB_PTP_TX_IN_PROGRESS, addr: &adapter->state);
1449	}
1450	}
1451
1452	/**
1453	* igb_ptp_stop - Disable PTP device and stop the overflow check.
1454	* @adapter: Board private structure.
1455	*
1456	* This function stops the PTP support and cancels the delayed work.
1457	**/
1458	void igb_ptp_stop(struct igb_adapter *adapter)
1459	{
1460	igb_ptp_suspend(adapter);
1461
1462	if (adapter->ptp_clock) {
1463	ptp_clock_unregister(ptp: adapter->ptp_clock);
1464	dev_info(&adapter->pdev->dev, "removed PHC on %s\n",
1465	adapter->netdev->name);
1466	adapter->ptp_flags &= ~IGB_PTP_ENABLED;
1467	}
1468	}
1469
1470	/**
1471	* igb_ptp_reset - Re-enable the adapter for PTP following a reset.
1472	* @adapter: Board private structure.
1473	*
1474	* This function handles the reset work required to re-enable the PTP device.
1475	**/
1476	void igb_ptp_reset(struct igb_adapter *adapter)
1477	{
1478	struct e1000_hw *hw = &adapter->hw;
1479	unsigned long flags;
1480
1481	/* reset the tstamp_config */
1482	igb_ptp_set_timestamp_mode(adapter, config: &adapter->tstamp_config);
1483
1484	spin_lock_irqsave(&adapter->tmreg_lock, flags);
1485
1486	switch (adapter->hw.mac.type) {
1487	case e1000_82576:
1488	/* Dial the nominal frequency. */
1489	wr32(E1000_TIMINCA, INCPERIOD_82576 | INCVALUE_82576);
1490	break;
1491	case e1000_82580:
1492	case e1000_i354:
1493	case e1000_i350:
1494	case e1000_i210:
1495	case e1000_i211:
1496	wr32(E1000_TSAUXC, 0x0);
1497	wr32(E1000_TSSDP, 0x0);
1498	wr32(E1000_TSIM,
1499	TSYNC_INTERRUPTS |
1500	(adapter->pps_sys_wrap_on ? TSINTR_SYS_WRAP : 0));
1501	wr32(E1000_IMS, E1000_IMS_TS);
1502	break;
1503	default:
1504	/* No work to do. */
1505	goto out;
1506	}
1507
1508	/* Re-initialize the timer. */
1509	if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211)) {
1510	struct timespec64 ts = ktime_to_timespec64(ktime_get_real());
1511
1512	igb_ptp_write_i210(adapter, ts: &ts);
1513	} else {
1514	timecounter_init(tc: &adapter->tc, cc: &adapter->cc,
1515	start_tstamp: ktime_to_ns(kt: ktime_get_real()));
1516	}
1517	out:
1518	spin_unlock_irqrestore(lock: &adapter->tmreg_lock, flags);
1519
1520	wrfl();
1521
1522	if (adapter->ptp_flags & IGB_PTP_OVERFLOW_CHECK)
1523	schedule_delayed_work(dwork: &adapter->ptp_overflow_work,
1524	IGB_SYSTIM_OVERFLOW_PERIOD);
1525	}
1526