1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* r8169.c: RealTek 8169/8168/8101 ethernet driver.
4	*
5	* Copyright (c) 2002 ShuChen <shuchen@realtek.com.tw>
6	* Copyright (c) 2003 - 2007 Francois Romieu <romieu@fr.zoreil.com>
7	* Copyright (c) a lot of people too. Please respect their work.
8	*
9	* See MAINTAINERS file for support contact information.
10	*/
11
12	#include <linux/module.h>
13	#include <linux/pci.h>
14	#include <linux/netdevice.h>
15	#include <linux/etherdevice.h>
16	#include <linux/clk.h>
17	#include <linux/delay.h>
18	#include <linux/ethtool.h>
19	#include <linux/phy.h>
20	#include <linux/if_vlan.h>
21	#include <linux/in.h>
22	#include <linux/io.h>
23	#include <linux/ip.h>
24	#include <linux/tcp.h>
25	#include <linux/interrupt.h>
26	#include <linux/dma-mapping.h>
27	#include <linux/pm_runtime.h>
28	#include <linux/bitfield.h>
29	#include <linux/prefetch.h>
30	#include <linux/ipv6.h>
31	#include <asm/unaligned.h>
32	#include <net/ip6_checksum.h>
33	#include <net/netdev_queues.h>
34
35	#include "r8169.h"
36	#include "r8169_firmware.h"
37
38	#define FIRMWARE_8168D_1 "rtl_nic/rtl8168d-1.fw"
39	#define FIRMWARE_8168D_2 "rtl_nic/rtl8168d-2.fw"
40	#define FIRMWARE_8168E_1 "rtl_nic/rtl8168e-1.fw"
41	#define FIRMWARE_8168E_2 "rtl_nic/rtl8168e-2.fw"
42	#define FIRMWARE_8168E_3 "rtl_nic/rtl8168e-3.fw"
43	#define FIRMWARE_8168F_1 "rtl_nic/rtl8168f-1.fw"
44	#define FIRMWARE_8168F_2 "rtl_nic/rtl8168f-2.fw"
45	#define FIRMWARE_8105E_1 "rtl_nic/rtl8105e-1.fw"
46	#define FIRMWARE_8402_1 "rtl_nic/rtl8402-1.fw"
47	#define FIRMWARE_8411_1 "rtl_nic/rtl8411-1.fw"
48	#define FIRMWARE_8411_2 "rtl_nic/rtl8411-2.fw"
49	#define FIRMWARE_8106E_1 "rtl_nic/rtl8106e-1.fw"
50	#define FIRMWARE_8106E_2 "rtl_nic/rtl8106e-2.fw"
51	#define FIRMWARE_8168G_2 "rtl_nic/rtl8168g-2.fw"
52	#define FIRMWARE_8168G_3 "rtl_nic/rtl8168g-3.fw"
53	#define FIRMWARE_8168H_2 "rtl_nic/rtl8168h-2.fw"
54	#define FIRMWARE_8168FP_3 "rtl_nic/rtl8168fp-3.fw"
55	#define FIRMWARE_8107E_2 "rtl_nic/rtl8107e-2.fw"
56	#define FIRMWARE_8125A_3 "rtl_nic/rtl8125a-3.fw"
57	#define FIRMWARE_8125B_2 "rtl_nic/rtl8125b-2.fw"
58	#define FIRMWARE_8126A_2 "rtl_nic/rtl8126a-2.fw"
59
60	#define TX_DMA_BURST 7 /* Maximum PCI burst, '7' is unlimited */
61	#define InterFrameGap 0x03 /* 3 means InterFrameGap = the shortest one */
62
63	#define R8169_REGS_SIZE 256
64	#define R8169_RX_BUF_SIZE (SZ_16K - 1)
65	#define NUM_TX_DESC 256 /* Number of Tx descriptor registers */
66	#define NUM_RX_DESC 256 /* Number of Rx descriptor registers */
67	#define R8169_TX_RING_BYTES (NUM_TX_DESC * sizeof(struct TxDesc))
68	#define R8169_RX_RING_BYTES (NUM_RX_DESC * sizeof(struct RxDesc))
69	#define R8169_TX_STOP_THRS (MAX_SKB_FRAGS + 1)
70	#define R8169_TX_START_THRS (2 * R8169_TX_STOP_THRS)
71
72	#define OCP_STD_PHY_BASE 0xa400
73
74	#define RTL_CFG_NO_GBIT 1
75
76	/* write/read MMIO register */
77	#define RTL_W8(tp, reg, val8) writeb((val8), tp->mmio_addr + (reg))
78	#define RTL_W16(tp, reg, val16) writew((val16), tp->mmio_addr + (reg))
79	#define RTL_W32(tp, reg, val32) writel((val32), tp->mmio_addr + (reg))
80	#define RTL_R8(tp, reg) readb(tp->mmio_addr + (reg))
81	#define RTL_R16(tp, reg) readw(tp->mmio_addr + (reg))
82	#define RTL_R32(tp, reg) readl(tp->mmio_addr + (reg))
83
84	#define JUMBO_4K (4 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
85	#define JUMBO_6K (6 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
86	#define JUMBO_7K (7 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
87	#define JUMBO_9K (9 * SZ_1K - VLAN_ETH_HLEN - ETH_FCS_LEN)
88
89	static const struct {
90	const char *name;
91	const char *fw_name;
92	} rtl_chip_infos[] = {
93	/* PCI devices. */
94	[RTL_GIGA_MAC_VER_02] = {.name: "RTL8169s" },
95	[RTL_GIGA_MAC_VER_03] = {.name: "RTL8110s" },
96	[RTL_GIGA_MAC_VER_04] = {.name: "RTL8169sb/8110sb" },
97	[RTL_GIGA_MAC_VER_05] = {.name: "RTL8169sc/8110sc" },
98	[RTL_GIGA_MAC_VER_06] = {.name: "RTL8169sc/8110sc" },
99	/* PCI-E devices. */
100	[RTL_GIGA_MAC_VER_07] = {.name: "RTL8102e" },
101	[RTL_GIGA_MAC_VER_08] = {.name: "RTL8102e" },
102	[RTL_GIGA_MAC_VER_09] = {.name: "RTL8102e/RTL8103e" },
103	[RTL_GIGA_MAC_VER_10] = {.name: "RTL8101e/RTL8100e" },
104	[RTL_GIGA_MAC_VER_11] = {.name: "RTL8168b/8111b" },
105	[RTL_GIGA_MAC_VER_14] = {.name: "RTL8401" },
106	[RTL_GIGA_MAC_VER_17] = {.name: "RTL8168b/8111b" },
107	[RTL_GIGA_MAC_VER_18] = {.name: "RTL8168cp/8111cp" },
108	[RTL_GIGA_MAC_VER_19] = {.name: "RTL8168c/8111c" },
109	[RTL_GIGA_MAC_VER_20] = {.name: "RTL8168c/8111c" },
110	[RTL_GIGA_MAC_VER_21] = {.name: "RTL8168c/8111c" },
111	[RTL_GIGA_MAC_VER_22] = {.name: "RTL8168c/8111c" },
112	[RTL_GIGA_MAC_VER_23] = {.name: "RTL8168cp/8111cp" },
113	[RTL_GIGA_MAC_VER_24] = {.name: "RTL8168cp/8111cp" },
114	[RTL_GIGA_MAC_VER_25] = {.name: "RTL8168d/8111d", FIRMWARE_8168D_1},
115	[RTL_GIGA_MAC_VER_26] = {.name: "RTL8168d/8111d", FIRMWARE_8168D_2},
116	[RTL_GIGA_MAC_VER_28] = {.name: "RTL8168dp/8111dp" },
117	[RTL_GIGA_MAC_VER_29] = {.name: "RTL8105e", FIRMWARE_8105E_1},
118	[RTL_GIGA_MAC_VER_30] = {.name: "RTL8105e", FIRMWARE_8105E_1},
119	[RTL_GIGA_MAC_VER_31] = {.name: "RTL8168dp/8111dp" },
120	[RTL_GIGA_MAC_VER_32] = {.name: "RTL8168e/8111e", FIRMWARE_8168E_1},
121	[RTL_GIGA_MAC_VER_33] = {.name: "RTL8168e/8111e", FIRMWARE_8168E_2},
122	[RTL_GIGA_MAC_VER_34] = {.name: "RTL8168evl/8111evl", FIRMWARE_8168E_3},
123	[RTL_GIGA_MAC_VER_35] = {.name: "RTL8168f/8111f", FIRMWARE_8168F_1},
124	[RTL_GIGA_MAC_VER_36] = {.name: "RTL8168f/8111f", FIRMWARE_8168F_2},
125	[RTL_GIGA_MAC_VER_37] = {.name: "RTL8402", FIRMWARE_8402_1 },
126	[RTL_GIGA_MAC_VER_38] = {.name: "RTL8411", FIRMWARE_8411_1 },
127	[RTL_GIGA_MAC_VER_39] = {.name: "RTL8106e", FIRMWARE_8106E_1},
128	[RTL_GIGA_MAC_VER_40] = {.name: "RTL8168g/8111g", FIRMWARE_8168G_2},
129	[RTL_GIGA_MAC_VER_42] = {.name: "RTL8168gu/8111gu", FIRMWARE_8168G_3},
130	[RTL_GIGA_MAC_VER_43] = {.name: "RTL8106eus", FIRMWARE_8106E_2},
131	[RTL_GIGA_MAC_VER_44] = {.name: "RTL8411b", FIRMWARE_8411_2 },
132	[RTL_GIGA_MAC_VER_46] = {.name: "RTL8168h/8111h", FIRMWARE_8168H_2},
133	[RTL_GIGA_MAC_VER_48] = {.name: "RTL8107e", FIRMWARE_8107E_2},
134	[RTL_GIGA_MAC_VER_51] = {.name: "RTL8168ep/8111ep" },
135	[RTL_GIGA_MAC_VER_52] = {.name: "RTL8168fp/RTL8117", FIRMWARE_8168FP_3},
136	[RTL_GIGA_MAC_VER_53] = {.name: "RTL8168fp/RTL8117", },
137	[RTL_GIGA_MAC_VER_61] = {.name: "RTL8125A", FIRMWARE_8125A_3},
138	/* reserve 62 for CFG_METHOD_4 in the vendor driver */
139	[RTL_GIGA_MAC_VER_63] = {.name: "RTL8125B", FIRMWARE_8125B_2},
140	[RTL_GIGA_MAC_VER_65] = {.name: "RTL8126A", FIRMWARE_8126A_2},
141	};
142
143	static const struct pci_device_id rtl8169_pci_tbl[] = {
144	{ PCI_VDEVICE(REALTEK, 0x2502) },
145	{ PCI_VDEVICE(REALTEK, 0x2600) },
146	{ PCI_VDEVICE(REALTEK, 0x8129) },
147	{ PCI_VDEVICE(REALTEK, 0x8136), RTL_CFG_NO_GBIT },
148	{ PCI_VDEVICE(REALTEK, 0x8161) },
149	{ PCI_VDEVICE(REALTEK, 0x8162) },
150	{ PCI_VDEVICE(REALTEK, 0x8167) },
151	{ PCI_VDEVICE(REALTEK, 0x8168) },
152	{ PCI_VDEVICE(NCUBE, 0x8168) },
153	{ PCI_VDEVICE(REALTEK, 0x8169) },
154	{ PCI_VENDOR_ID_DLINK, 0x4300,
155	PCI_VENDOR_ID_DLINK, 0x4b10, 0, 0 },
156	{ PCI_VDEVICE(DLINK, 0x4300) },
157	{ PCI_VDEVICE(DLINK, 0x4302) },
158	{ PCI_VDEVICE(AT, 0xc107) },
159	{ PCI_VDEVICE(USR, 0x0116) },
160	{ PCI_VENDOR_ID_LINKSYS, 0x1032, PCI_ANY_ID, 0x0024 },
161	{ 0x0001, 0x8168, PCI_ANY_ID, 0x2410 },
162	{ PCI_VDEVICE(REALTEK, 0x8125) },
163	{ PCI_VDEVICE(REALTEK, 0x8126) },
164	{ PCI_VDEVICE(REALTEK, 0x3000) },
165	{}
166	};
167
168	MODULE_DEVICE_TABLE(pci, rtl8169_pci_tbl);
169
170	enum rtl_registers {
171	MAC0 = 0, /* Ethernet hardware address. */
172	MAC4 = 4,
173	MAR0 = 8, /* Multicast filter. */
174	CounterAddrLow = 0x10,
175	CounterAddrHigh = 0x14,
176	TxDescStartAddrLow = 0x20,
177	TxDescStartAddrHigh = 0x24,
178	TxHDescStartAddrLow = 0x28,
179	TxHDescStartAddrHigh = 0x2c,
180	FLASH = 0x30,
181	ERSR = 0x36,
182	ChipCmd = 0x37,
183	TxPoll = 0x38,
184	IntrMask = 0x3c,
185	IntrStatus = 0x3e,
186
187	TxConfig = 0x40,
188	#define TXCFG_AUTO_FIFO (1 << 7) /* 8111e-vl */
189	#define TXCFG_EMPTY (1 << 11) /* 8111e-vl */
190
191	RxConfig = 0x44,
192	#define RX128_INT_EN (1 << 15) /* 8111c and later */
193	#define RX_MULTI_EN (1 << 14) /* 8111c only */
194	#define RXCFG_FIFO_SHIFT 13
195	/* No threshold before first PCI xfer */
196	#define RX_FIFO_THRESH (7 << RXCFG_FIFO_SHIFT)
197	#define RX_EARLY_OFF (1 << 11)
198	#define RX_PAUSE_SLOT_ON (1 << 11) /* 8125b and later */
199	#define RXCFG_DMA_SHIFT 8
200	/* Unlimited maximum PCI burst. */
201	#define RX_DMA_BURST (7 << RXCFG_DMA_SHIFT)
202
203	Cfg9346 = 0x50,
204	Config0 = 0x51,
205	Config1 = 0x52,
206	Config2 = 0x53,
207	#define PME_SIGNAL (1 << 5) /* 8168c and later */
208
209	Config3 = 0x54,
210	Config4 = 0x55,
211	Config5 = 0x56,
212	PHYAR = 0x60,
213	PHYstatus = 0x6c,
214	RxMaxSize = 0xda,
215	CPlusCmd = 0xe0,
216	IntrMitigate = 0xe2,
217
218	#define RTL_COALESCE_TX_USECS GENMASK(15, 12)
219	#define RTL_COALESCE_TX_FRAMES GENMASK(11, 8)
220	#define RTL_COALESCE_RX_USECS GENMASK(7, 4)
221	#define RTL_COALESCE_RX_FRAMES GENMASK(3, 0)
222
223	#define RTL_COALESCE_T_MAX 0x0fU
224	#define RTL_COALESCE_FRAME_MAX (RTL_COALESCE_T_MAX * 4)
225
226	RxDescAddrLow = 0xe4,
227	RxDescAddrHigh = 0xe8,
228	EarlyTxThres = 0xec, /* 8169. Unit of 32 bytes. */
229
230	#define NoEarlyTx 0x3f /* Max value : no early transmit. */
231
232	MaxTxPacketSize = 0xec, /* 8101/8168. Unit of 128 bytes. */
233
234	#define TxPacketMax (8064 >> 7)
235	#define EarlySize 0x27
236
237	FuncEvent = 0xf0,
238	FuncEventMask = 0xf4,
239	FuncPresetState = 0xf8,
240	IBCR0 = 0xf8,
241	IBCR2 = 0xf9,
242	IBIMR0 = 0xfa,
243	IBISR0 = 0xfb,
244	FuncForceEvent = 0xfc,
245	};
246
247	enum rtl8168_8101_registers {
248	CSIDR = 0x64,
249	CSIAR = 0x68,
250	#define CSIAR_FLAG 0x80000000
251	#define CSIAR_WRITE_CMD 0x80000000
252	#define CSIAR_BYTE_ENABLE 0x0000f000
253	#define CSIAR_ADDR_MASK 0x00000fff
254	PMCH = 0x6f,
255	#define D3COLD_NO_PLL_DOWN BIT(7)
256	#define D3HOT_NO_PLL_DOWN BIT(6)
257	#define D3_NO_PLL_DOWN (BIT(7) | BIT(6))
258	EPHYAR = 0x80,
259	#define EPHYAR_FLAG 0x80000000
260	#define EPHYAR_WRITE_CMD 0x80000000
261	#define EPHYAR_REG_MASK 0x1f
262	#define EPHYAR_REG_SHIFT 16
263	#define EPHYAR_DATA_MASK 0xffff
264	DLLPR = 0xd0,
265	#define PFM_EN (1 << 6)
266	#define TX_10M_PS_EN (1 << 7)
267	DBG_REG = 0xd1,
268	#define FIX_NAK_1 (1 << 4)
269	#define FIX_NAK_2 (1 << 3)
270	TWSI = 0xd2,
271	MCU = 0xd3,
272	#define NOW_IS_OOB (1 << 7)
273	#define TX_EMPTY (1 << 5)
274	#define RX_EMPTY (1 << 4)
275	#define RXTX_EMPTY (TX_EMPTY | RX_EMPTY)
276	#define EN_NDP (1 << 3)
277	#define EN_OOB_RESET (1 << 2)
278	#define LINK_LIST_RDY (1 << 1)
279	EFUSEAR = 0xdc,
280	#define EFUSEAR_FLAG 0x80000000
281	#define EFUSEAR_WRITE_CMD 0x80000000
282	#define EFUSEAR_READ_CMD 0x00000000
283	#define EFUSEAR_REG_MASK 0x03ff
284	#define EFUSEAR_REG_SHIFT 8
285	#define EFUSEAR_DATA_MASK 0xff
286	MISC_1 = 0xf2,
287	#define PFM_D3COLD_EN (1 << 6)
288	};
289
290	enum rtl8168_registers {
291	LED_CTRL = 0x18,
292	LED_FREQ = 0x1a,
293	EEE_LED = 0x1b,
294	ERIDR = 0x70,
295	ERIAR = 0x74,
296	#define ERIAR_FLAG 0x80000000
297	#define ERIAR_WRITE_CMD 0x80000000
298	#define ERIAR_READ_CMD 0x00000000
299	#define ERIAR_ADDR_BYTE_ALIGN 4
300	#define ERIAR_TYPE_SHIFT 16
301	#define ERIAR_EXGMAC (0x00 << ERIAR_TYPE_SHIFT)
302	#define ERIAR_MSIX (0x01 << ERIAR_TYPE_SHIFT)
303	#define ERIAR_ASF (0x02 << ERIAR_TYPE_SHIFT)
304	#define ERIAR_OOB (0x02 << ERIAR_TYPE_SHIFT)
305	#define ERIAR_MASK_SHIFT 12
306	#define ERIAR_MASK_0001 (0x1 << ERIAR_MASK_SHIFT)
307	#define ERIAR_MASK_0011 (0x3 << ERIAR_MASK_SHIFT)
308	#define ERIAR_MASK_0100 (0x4 << ERIAR_MASK_SHIFT)
309	#define ERIAR_MASK_0101 (0x5 << ERIAR_MASK_SHIFT)
310	#define ERIAR_MASK_1111 (0xf << ERIAR_MASK_SHIFT)
311	EPHY_RXER_NUM = 0x7c,
312	OCPDR = 0xb0, /* OCP GPHY access */
313	#define OCPDR_WRITE_CMD 0x80000000
314	#define OCPDR_READ_CMD 0x00000000
315	#define OCPDR_REG_MASK 0x7f
316	#define OCPDR_GPHY_REG_SHIFT 16
317	#define OCPDR_DATA_MASK 0xffff
318	OCPAR = 0xb4,
319	#define OCPAR_FLAG 0x80000000
320	#define OCPAR_GPHY_WRITE_CMD 0x8000f060
321	#define OCPAR_GPHY_READ_CMD 0x0000f060
322	GPHY_OCP = 0xb8,
323	RDSAR1 = 0xd0, /* 8168c only. Undocumented on 8168dp */
324	MISC = 0xf0, /* 8168e only. */
325	#define TXPLA_RST (1 << 29)
326	#define DISABLE_LAN_EN (1 << 23) /* Enable GPIO pin */
327	#define PWM_EN (1 << 22)
328	#define RXDV_GATED_EN (1 << 19)
329	#define EARLY_TALLY_EN (1 << 16)
330	};
331
332	enum rtl8125_registers {
333	LEDSEL0 = 0x18,
334	INT_CFG0_8125 = 0x34,
335	#define INT_CFG0_ENABLE_8125 BIT(0)
336	#define INT_CFG0_CLKREQEN BIT(3)
337	IntrMask_8125 = 0x38,
338	IntrStatus_8125 = 0x3c,
339	INT_CFG1_8125 = 0x7a,
340	LEDSEL2 = 0x84,
341	LEDSEL1 = 0x86,
342	TxPoll_8125 = 0x90,
343	LEDSEL3 = 0x96,
344	MAC0_BKP = 0x19e0,
345	EEE_TXIDLE_TIMER_8125 = 0x6048,
346	};
347
348	#define LEDSEL_MASK_8125 0x23f
349
350	#define RX_VLAN_INNER_8125 BIT(22)
351	#define RX_VLAN_OUTER_8125 BIT(23)
352	#define RX_VLAN_8125 (RX_VLAN_INNER_8125 | RX_VLAN_OUTER_8125)
353
354	#define RX_FETCH_DFLT_8125 (8 << 27)
355
356	enum rtl_register_content {
357	/* InterruptStatusBits */
358	SYSErr = 0x8000,
359	PCSTimeout = 0x4000,
360	SWInt = 0x0100,
361	TxDescUnavail = 0x0080,
362	RxFIFOOver = 0x0040,
363	LinkChg = 0x0020,
364	RxOverflow = 0x0010,
365	TxErr = 0x0008,
366	TxOK = 0x0004,
367	RxErr = 0x0002,
368	RxOK = 0x0001,
369
370	/* RxStatusDesc */
371	RxRWT = (1 << 22),
372	RxRES = (1 << 21),
373	RxRUNT = (1 << 20),
374	RxCRC = (1 << 19),
375
376	/* ChipCmdBits */
377	StopReq = 0x80,
378	CmdReset = 0x10,
379	CmdRxEnb = 0x08,
380	CmdTxEnb = 0x04,
381	RxBufEmpty = 0x01,
382
383	/* TXPoll register p.5 */
384	HPQ = 0x80, /* Poll cmd on the high prio queue */
385	NPQ = 0x40, /* Poll cmd on the low prio queue */
386	FSWInt = 0x01, /* Forced software interrupt */
387
388	/* Cfg9346Bits */
389	Cfg9346_Lock = 0x00,
390	Cfg9346_Unlock = 0xc0,
391
392	/* rx_mode_bits */
393	AcceptErr = 0x20,
394	AcceptRunt = 0x10,
395	#define RX_CONFIG_ACCEPT_ERR_MASK 0x30
396	AcceptBroadcast = 0x08,
397	AcceptMulticast = 0x04,
398	AcceptMyPhys = 0x02,
399	AcceptAllPhys = 0x01,
400	#define RX_CONFIG_ACCEPT_OK_MASK 0x0f
401	#define RX_CONFIG_ACCEPT_MASK 0x3f
402
403	/* TxConfigBits */
404	TxInterFrameGapShift = 24,
405	TxDMAShift = 8, /* DMA burst value (0-7) is shift this many bits */
406
407	/* Config1 register p.24 */
408	LEDS1 = (1 << 7),
409	LEDS0 = (1 << 6),
410	Speed_down = (1 << 4),
411	MEMMAP = (1 << 3),
412	IOMAP = (1 << 2),
413	VPD = (1 << 1),
414	PMEnable = (1 << 0), /* Power Management Enable */
415
416	/* Config2 register p. 25 */
417	ClkReqEn = (1 << 7), /* Clock Request Enable */
418	MSIEnable = (1 << 5), /* 8169 only. Reserved in the 8168. */
419	PCI_Clock_66MHz = 0x01,
420	PCI_Clock_33MHz = 0x00,
421
422	/* Config3 register p.25 */
423	MagicPacket = (1 << 5), /* Wake up when receives a Magic Packet */
424	LinkUp = (1 << 4), /* Wake up when the cable connection is re-established */
425	Jumbo_En0 = (1 << 2), /* 8168 only. Reserved in the 8168b */
426	Rdy_to_L23 = (1 << 1), /* L23 Enable */
427	Beacon_en = (1 << 0), /* 8168 only. Reserved in the 8168b */
428
429	/* Config4 register */
430	Jumbo_En1 = (1 << 1), /* 8168 only. Reserved in the 8168b */
431
432	/* Config5 register p.27 */
433	BWF = (1 << 6), /* Accept Broadcast wakeup frame */
434	MWF = (1 << 5), /* Accept Multicast wakeup frame */
435	UWF = (1 << 4), /* Accept Unicast wakeup frame */
436	Spi_en = (1 << 3),
437	LanWake = (1 << 1), /* LanWake enable/disable */
438	PMEStatus = (1 << 0), /* PME status can be reset by PCI RST# */
439	ASPM_en = (1 << 0), /* ASPM enable */
440
441	/* CPlusCmd p.31 */
442	EnableBist = (1 << 15), // 8168 8101
443	Mac_dbgo_oe = (1 << 14), // 8168 8101
444	EnAnaPLL = (1 << 14), // 8169
445	Normal_mode = (1 << 13), // unused
446	Force_half_dup = (1 << 12), // 8168 8101
447	Force_rxflow_en = (1 << 11), // 8168 8101
448	Force_txflow_en = (1 << 10), // 8168 8101
449	Cxpl_dbg_sel = (1 << 9), // 8168 8101
450	ASF = (1 << 8), // 8168 8101
451	PktCntrDisable = (1 << 7), // 8168 8101
452	Mac_dbgo_sel = 0x001c, // 8168
453	RxVlan = (1 << 6),
454	RxChkSum = (1 << 5),
455	PCIDAC = (1 << 4),
456	PCIMulRW = (1 << 3),
457	#define INTT_MASK GENMASK(1, 0)
458	#define CPCMD_MASK (Normal_mode | RxVlan | RxChkSum | INTT_MASK)
459
460	/* rtl8169_PHYstatus */
461	TBI_Enable = 0x80,
462	TxFlowCtrl = 0x40,
463	RxFlowCtrl = 0x20,
464	_1000bpsF = 0x10,
465	_100bps = 0x08,
466	_10bps = 0x04,
467	LinkStatus = 0x02,
468	FullDup = 0x01,
469
470	/* ResetCounterCommand */
471	CounterReset = 0x1,
472
473	/* DumpCounterCommand */
474	CounterDump = 0x8,
475
476	/* magic enable v2 */
477	MagicPacket_v2 = (1 << 16), /* Wake up when receives a Magic Packet */
478	};
479
480	enum rtl_desc_bit {
481	/* First doubleword. */
482	DescOwn = (1 << 31), /* Descriptor is owned by NIC */
483	RingEnd = (1 << 30), /* End of descriptor ring */
484	FirstFrag = (1 << 29), /* First segment of a packet */
485	LastFrag = (1 << 28), /* Final segment of a packet */
486	};
487
488	/* Generic case. */
489	enum rtl_tx_desc_bit {
490	/* First doubleword. */
491	TD_LSO = (1 << 27), /* Large Send Offload */
492	#define TD_MSS_MAX 0x07ffu /* MSS value */
493
494	/* Second doubleword. */
495	TxVlanTag = (1 << 17), /* Add VLAN tag */
496	};
497
498	/* 8169, 8168b and 810x except 8102e. */
499	enum rtl_tx_desc_bit_0 {
500	/* First doubleword. */
501	#define TD0_MSS_SHIFT 16 /* MSS position (11 bits) */
502	TD0_TCP_CS = (1 << 16), /* Calculate TCP/IP checksum */
503	TD0_UDP_CS = (1 << 17), /* Calculate UDP/IP checksum */
504	TD0_IP_CS = (1 << 18), /* Calculate IP checksum */
505	};
506
507	/* 8102e, 8168c and beyond. */
508	enum rtl_tx_desc_bit_1 {
509	/* First doubleword. */
510	TD1_GTSENV4 = (1 << 26), /* Giant Send for IPv4 */
511	TD1_GTSENV6 = (1 << 25), /* Giant Send for IPv6 */
512	#define GTTCPHO_SHIFT 18
513	#define GTTCPHO_MAX 0x7f
514
515	/* Second doubleword. */
516	#define TCPHO_SHIFT 18
517	#define TCPHO_MAX 0x3ff
518	#define TD1_MSS_SHIFT 18 /* MSS position (11 bits) */
519	TD1_IPv6_CS = (1 << 28), /* Calculate IPv6 checksum */
520	TD1_IPv4_CS = (1 << 29), /* Calculate IPv4 checksum */
521	TD1_TCP_CS = (1 << 30), /* Calculate TCP/IP checksum */
522	TD1_UDP_CS = (1 << 31), /* Calculate UDP/IP checksum */
523	};
524
525	enum rtl_rx_desc_bit {
526	/* Rx private */
527	PID1 = (1 << 18), /* Protocol ID bit 1/2 */
528	PID0 = (1 << 17), /* Protocol ID bit 0/2 */
529
530	#define RxProtoUDP (PID1)
531	#define RxProtoTCP (PID0)
532	#define RxProtoIP (PID1 | PID0)
533	#define RxProtoMask RxProtoIP
534
535	IPFail = (1 << 16), /* IP checksum failed */
536	UDPFail = (1 << 15), /* UDP/IP checksum failed */
537	TCPFail = (1 << 14), /* TCP/IP checksum failed */
538
539	#define RxCSFailMask (IPFail | UDPFail | TCPFail)
540
541	RxVlanTag = (1 << 16), /* VLAN tag available */
542	};
543
544	#define RTL_GSO_MAX_SIZE_V1 32000
545	#define RTL_GSO_MAX_SEGS_V1 24
546	#define RTL_GSO_MAX_SIZE_V2 64000
547	#define RTL_GSO_MAX_SEGS_V2 64
548
549	struct TxDesc {
550	__le32 opts1;
551	__le32 opts2;
552	__le64 addr;
553	};
554
555	struct RxDesc {
556	__le32 opts1;
557	__le32 opts2;
558	__le64 addr;
559	};
560
561	struct ring_info {
562	struct sk_buff *skb;
563	u32 len;
564	};
565
566	struct rtl8169_counters {
567	__le64 tx_packets;
568	__le64 rx_packets;
569	__le64 tx_errors;
570	__le32 rx_errors;
571	__le16 rx_missed;
572	__le16 align_errors;
573	__le32 tx_one_collision;
574	__le32 tx_multi_collision;
575	__le64 rx_unicast;
576	__le64 rx_broadcast;
577	__le32 rx_multicast;
578	__le16 tx_aborted;
579	__le16 tx_underun;
580	};
581
582	struct rtl8169_tc_offsets {
583	bool inited;
584	__le64 tx_errors;
585	__le32 tx_multi_collision;
586	__le16 tx_aborted;
587	__le16 rx_missed;
588	};
589
590	enum rtl_flag {
591	RTL_FLAG_TASK_ENABLED = 0,
592	RTL_FLAG_TASK_RESET_PENDING,
593	RTL_FLAG_TASK_RESET_NO_QUEUE_WAKE,
594	RTL_FLAG_TASK_TX_TIMEOUT,
595	RTL_FLAG_MAX
596	};
597
598	enum rtl_dash_type {
599	RTL_DASH_NONE,
600	RTL_DASH_DP,
601	RTL_DASH_EP,
602	};
603
604	struct rtl8169_private {
605	void __iomem *mmio_addr; /* memory map physical address */
606	struct pci_dev *pci_dev;
607	struct net_device *dev;
608	struct phy_device *phydev;
609	struct napi_struct napi;
610	enum mac_version mac_version;
611	enum rtl_dash_type dash_type;
612	u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */
613	u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */
614	u32 dirty_tx;
615	struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */
616	struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */
617	dma_addr_t TxPhyAddr;
618	dma_addr_t RxPhyAddr;
619	struct page *Rx_databuff[NUM_RX_DESC]; /* Rx data buffers */
620	struct ring_info tx_skb[NUM_TX_DESC]; /* Tx data buffers */
621	u16 cp_cmd;
622	u16 tx_lpi_timer;
623	u32 irq_mask;
624	int irq;
625	struct clk *clk;
626
627	struct {
628	DECLARE_BITMAP(flags, RTL_FLAG_MAX);
629	struct work_struct work;
630	} wk;
631
632	raw_spinlock_t config25_lock;
633	raw_spinlock_t mac_ocp_lock;
634	struct mutex led_lock; /* serialize LED ctrl RMW access */
635
636	raw_spinlock_t cfg9346_usage_lock;
637	int cfg9346_usage_count;
638
639	unsigned supports_gmii:1;
640	unsigned aspm_manageable:1;
641	unsigned dash_enabled:1;
642	dma_addr_t counters_phys_addr;
643	struct rtl8169_counters *counters;
644	struct rtl8169_tc_offsets tc_offset;
645	u32 saved_wolopts;
646
647	const char *fw_name;
648	struct rtl_fw *rtl_fw;
649
650	struct r8169_led_classdev *leds;
651
652	u32 ocp_base;
653	};
654
655	typedef void (*rtl_generic_fct)(struct rtl8169_private *tp);
656
657	MODULE_AUTHOR("Realtek and the Linux r8169 crew <netdev@vger.kernel.org>");
658	MODULE_DESCRIPTION("RealTek RTL-8169 Gigabit Ethernet driver");
659	MODULE_SOFTDEP("pre: realtek");
660	MODULE_LICENSE("GPL");
661	MODULE_FIRMWARE(FIRMWARE_8168D_1);
662	MODULE_FIRMWARE(FIRMWARE_8168D_2);
663	MODULE_FIRMWARE(FIRMWARE_8168E_1);
664	MODULE_FIRMWARE(FIRMWARE_8168E_2);
665	MODULE_FIRMWARE(FIRMWARE_8168E_3);
666	MODULE_FIRMWARE(FIRMWARE_8105E_1);
667	MODULE_FIRMWARE(FIRMWARE_8168F_1);
668	MODULE_FIRMWARE(FIRMWARE_8168F_2);
669	MODULE_FIRMWARE(FIRMWARE_8402_1);
670	MODULE_FIRMWARE(FIRMWARE_8411_1);
671	MODULE_FIRMWARE(FIRMWARE_8411_2);
672	MODULE_FIRMWARE(FIRMWARE_8106E_1);
673	MODULE_FIRMWARE(FIRMWARE_8106E_2);
674	MODULE_FIRMWARE(FIRMWARE_8168G_2);
675	MODULE_FIRMWARE(FIRMWARE_8168G_3);
676	MODULE_FIRMWARE(FIRMWARE_8168H_2);
677	MODULE_FIRMWARE(FIRMWARE_8168FP_3);
678	MODULE_FIRMWARE(FIRMWARE_8107E_2);
679	MODULE_FIRMWARE(FIRMWARE_8125A_3);
680	MODULE_FIRMWARE(FIRMWARE_8125B_2);
681	MODULE_FIRMWARE(FIRMWARE_8126A_2);
682
683	static inline struct device *tp_to_dev(struct rtl8169_private *tp)
684	{
685	return &tp->pci_dev->dev;
686	}
687
688	static void rtl_lock_config_regs(struct rtl8169_private *tp)
689	{
690	unsigned long flags;
691
692	raw_spin_lock_irqsave(&tp->cfg9346_usage_lock, flags);
693	if (!--tp->cfg9346_usage_count)
694	RTL_W8(tp, Cfg9346, Cfg9346_Lock);
695	raw_spin_unlock_irqrestore(&tp->cfg9346_usage_lock, flags);
696	}
697
698	static void rtl_unlock_config_regs(struct rtl8169_private *tp)
699	{
700	unsigned long flags;
701
702	raw_spin_lock_irqsave(&tp->cfg9346_usage_lock, flags);
703	if (!tp->cfg9346_usage_count++)
704	RTL_W8(tp, Cfg9346, Cfg9346_Unlock);
705	raw_spin_unlock_irqrestore(&tp->cfg9346_usage_lock, flags);
706	}
707
708	static void rtl_pci_commit(struct rtl8169_private *tp)
709	{
710	/* Read an arbitrary register to commit a preceding PCI write */
711	RTL_R8(tp, ChipCmd);
712	}
713
714	static void rtl_mod_config2(struct rtl8169_private *tp, u8 clear, u8 set)
715	{
716	unsigned long flags;
717	u8 val;
718
719	raw_spin_lock_irqsave(&tp->config25_lock, flags);
720	val = RTL_R8(tp, Config2);
721	RTL_W8(tp, Config2, (val & ~clear) | set);
722	raw_spin_unlock_irqrestore(&tp->config25_lock, flags);
723	}
724
725	static void rtl_mod_config5(struct rtl8169_private *tp, u8 clear, u8 set)
726	{
727	unsigned long flags;
728	u8 val;
729
730	raw_spin_lock_irqsave(&tp->config25_lock, flags);
731	val = RTL_R8(tp, Config5);
732	RTL_W8(tp, Config5, (val & ~clear) | set);
733	raw_spin_unlock_irqrestore(&tp->config25_lock, flags);
734	}
735
736	static bool rtl_is_8125(struct rtl8169_private *tp)
737	{
738	return tp->mac_version >= RTL_GIGA_MAC_VER_61;
739	}
740
741	static bool rtl_is_8168evl_up(struct rtl8169_private *tp)
742	{
743	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
744	tp->mac_version != RTL_GIGA_MAC_VER_39 &&
745	tp->mac_version <= RTL_GIGA_MAC_VER_53;
746	}
747
748	static bool rtl_supports_eee(struct rtl8169_private *tp)
749	{
750	return tp->mac_version >= RTL_GIGA_MAC_VER_34 &&
751	tp->mac_version != RTL_GIGA_MAC_VER_37 &&
752	tp->mac_version != RTL_GIGA_MAC_VER_39;
753	}
754
755	static void rtl_read_mac_from_reg(struct rtl8169_private *tp, u8 *mac, int reg)
756	{
757	int i;
758
759	for (i = 0; i < ETH_ALEN; i++)
760	mac[i] = RTL_R8(tp, reg + i);
761	}
762
763	struct rtl_cond {
764	bool (*check)(struct rtl8169_private *);
765	const char *msg;
766	};
767
768	static bool rtl_loop_wait(struct rtl8169_private *tp, const struct rtl_cond *c,
769	unsigned long usecs, int n, bool high)
770	{
771	int i;
772
773	for (i = 0; i < n; i++) {
774	if (c->check(tp) == high)
775	return true;
776	fsleep(usecs);
777	}
778
779	if (net_ratelimit())
780	netdev_err(dev: tp->dev, format: "%s == %d (loop: %d, delay: %lu).\n",
781	c->msg, !high, n, usecs);
782	return false;
783	}
784
785	static bool rtl_loop_wait_high(struct rtl8169_private *tp,
786	const struct rtl_cond *c,
787	unsigned long d, int n)
788	{
789	return rtl_loop_wait(tp, c, usecs: d, n, high: true);
790	}
791
792	static bool rtl_loop_wait_low(struct rtl8169_private *tp,
793	const struct rtl_cond *c,
794	unsigned long d, int n)
795	{
796	return rtl_loop_wait(tp, c, usecs: d, n, high: false);
797	}
798
799	#define DECLARE_RTL_COND(name) \
800	static bool name ## _check(struct rtl8169_private *); \
801	\
802	static const struct rtl_cond name = { \
803	.check = name ## _check, \
804	.msg = #name \
805	}; \
806	\
807	static bool name ## _check(struct rtl8169_private *tp)
808
809	int rtl8168_led_mod_ctrl(struct rtl8169_private *tp, u16 mask, u16 val)
810	{
811	struct device *dev = tp_to_dev(tp);
812	int ret;
813
814	ret = pm_runtime_resume_and_get(dev);
815	if (ret < 0)
816	return ret;
817
818	mutex_lock(&tp->led_lock);
819	RTL_W16(tp, LED_CTRL, (RTL_R16(tp, LED_CTRL) & ~mask) | val);
820	mutex_unlock(lock: &tp->led_lock);
821
822	pm_runtime_put_sync(dev);
823
824	return 0;
825	}
826
827	int rtl8168_get_led_mode(struct rtl8169_private *tp)
828	{
829	struct device *dev = tp_to_dev(tp);
830	int ret;
831
832	ret = pm_runtime_resume_and_get(dev);
833	if (ret < 0)
834	return ret;
835
836	ret = RTL_R16(tp, LED_CTRL);
837
838	pm_runtime_put_sync(dev);
839
840	return ret;
841	}
842
843	static int rtl8125_get_led_reg(int index)
844	{
845	static const int led_regs[] = { LEDSEL0, LEDSEL1, LEDSEL2, LEDSEL3 };
846
847	return led_regs[index];
848	}
849
850	int rtl8125_set_led_mode(struct rtl8169_private *tp, int index, u16 mode)
851	{
852	int reg = rtl8125_get_led_reg(index);
853	struct device *dev = tp_to_dev(tp);
854	int ret;
855	u16 val;
856
857	ret = pm_runtime_resume_and_get(dev);
858	if (ret < 0)
859	return ret;
860
861	mutex_lock(&tp->led_lock);
862	val = RTL_R16(tp, reg) & ~LEDSEL_MASK_8125;
863	RTL_W16(tp, reg, val | mode);
864	mutex_unlock(lock: &tp->led_lock);
865
866	pm_runtime_put_sync(dev);
867
868	return 0;
869	}
870
871	int rtl8125_get_led_mode(struct rtl8169_private *tp, int index)
872	{
873	int reg = rtl8125_get_led_reg(index);
874	struct device *dev = tp_to_dev(tp);
875	int ret;
876
877	ret = pm_runtime_resume_and_get(dev);
878	if (ret < 0)
879	return ret;
880
881	ret = RTL_R16(tp, reg);
882
883	pm_runtime_put_sync(dev);
884
885	return ret;
886	}
887
888	void r8169_get_led_name(struct rtl8169_private *tp, int idx,
889	char *buf, int buf_len)
890	{
891	struct pci_dev *pdev = tp->pci_dev;
892	char pdom[8], pfun[8];
893	int domain;
894
895	domain = pci_domain_nr(bus: pdev->bus);
896	if (domain)
897	snprintf(buf: pdom, size: sizeof(pdom), fmt: "P%d", domain);
898	else
899	pdom[0] = '\0';
900
901	if (pdev->multifunction)
902	snprintf(buf: pfun, size: sizeof(pfun), fmt: "f%d", PCI_FUNC(pdev->devfn));
903	else
904	pfun[0] = '\0';
905
906	snprintf(buf, size: buf_len, fmt: "en%sp%ds%d%s-%d::lan", pdom, pdev->bus->number,
907	PCI_SLOT(pdev->devfn), pfun, idx);
908	}
909
910	static void r8168fp_adjust_ocp_cmd(struct rtl8169_private *tp, u32 *cmd, int type)
911	{
912	/* based on RTL8168FP_OOBMAC_BASE in vendor driver */
913	if (type == ERIAR_OOB &&
914	(tp->mac_version == RTL_GIGA_MAC_VER_52 ||
915	tp->mac_version == RTL_GIGA_MAC_VER_53))
916	*cmd |= 0xf70 << 18;
917	}
918
919	DECLARE_RTL_COND(rtl_eriar_cond)
920	{
921	return RTL_R32(tp, ERIAR) & ERIAR_FLAG;
922	}
923
924	static void _rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
925	u32 val, int type)
926	{
927	u32 cmd = ERIAR_WRITE_CMD | type | mask | addr;
928
929	if (WARN(addr & 3 || !mask, "addr: 0x%x, mask: 0x%08x\n", addr, mask))
930	return;
931
932	RTL_W32(tp, ERIDR, val);
933	r8168fp_adjust_ocp_cmd(tp, cmd: &cmd, type);
934	RTL_W32(tp, ERIAR, cmd);
935
936	rtl_loop_wait_low(tp, c: &rtl_eriar_cond, d: 100, n: 100);
937	}
938
939	static void rtl_eri_write(struct rtl8169_private *tp, int addr, u32 mask,
940	u32 val)
941	{
942	_rtl_eri_write(tp, addr, mask, val, ERIAR_EXGMAC);
943	}
944
945	static u32 _rtl_eri_read(struct rtl8169_private *tp, int addr, int type)
946	{
947	u32 cmd = ERIAR_READ_CMD | type | ERIAR_MASK_1111 | addr;
948
949	r8168fp_adjust_ocp_cmd(tp, cmd: &cmd, type);
950	RTL_W32(tp, ERIAR, cmd);
951
952	return rtl_loop_wait_high(tp, c: &rtl_eriar_cond, d: 100, n: 100) ?
953	RTL_R32(tp, ERIDR) : ~0;
954	}
955
956	static u32 rtl_eri_read(struct rtl8169_private *tp, int addr)
957	{
958	return _rtl_eri_read(tp, addr, ERIAR_EXGMAC);
959	}
960
961	static void rtl_w0w1_eri(struct rtl8169_private *tp, int addr, u32 p, u32 m)
962	{
963	u32 val = rtl_eri_read(tp, addr);
964
965	rtl_eri_write(tp, addr, ERIAR_MASK_1111, val: (val & ~m) | p);
966	}
967
968	static void rtl_eri_set_bits(struct rtl8169_private *tp, int addr, u32 p)
969	{
970	rtl_w0w1_eri(tp, addr, p, m: 0);
971	}
972
973	static void rtl_eri_clear_bits(struct rtl8169_private *tp, int addr, u32 m)
974	{
975	rtl_w0w1_eri(tp, addr, p: 0, m);
976	}
977
978	static bool rtl_ocp_reg_failure(u32 reg)
979	{
980	return WARN_ONCE(reg & 0xffff0001, "Invalid ocp reg %x!\n", reg);
981	}
982
983	DECLARE_RTL_COND(rtl_ocp_gphy_cond)
984	{
985	return RTL_R32(tp, GPHY_OCP) & OCPAR_FLAG;
986	}
987
988	static void r8168_phy_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
989	{
990	if (rtl_ocp_reg_failure(reg))
991	return;
992
993	RTL_W32(tp, GPHY_OCP, OCPAR_FLAG | (reg << 15) | data);
994
995	rtl_loop_wait_low(tp, c: &rtl_ocp_gphy_cond, d: 25, n: 10);
996	}
997
998	static int r8168_phy_ocp_read(struct rtl8169_private *tp, u32 reg)
999	{
1000	if (rtl_ocp_reg_failure(reg))
1001	return 0;
1002
1003	RTL_W32(tp, GPHY_OCP, reg << 15);
1004
1005	return rtl_loop_wait_high(tp, c: &rtl_ocp_gphy_cond, d: 25, n: 10) ?
1006	(RTL_R32(tp, GPHY_OCP) & 0xffff) : -ETIMEDOUT;
1007	}
1008
1009	static void __r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
1010	{
1011	if (rtl_ocp_reg_failure(reg))
1012	return;
1013
1014	RTL_W32(tp, OCPDR, OCPAR_FLAG | (reg << 15) | data);
1015	}
1016
1017	static void r8168_mac_ocp_write(struct rtl8169_private *tp, u32 reg, u32 data)
1018	{
1019	unsigned long flags;
1020
1021	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
1022	__r8168_mac_ocp_write(tp, reg, data);
1023	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
1024	}
1025
1026	static u16 __r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
1027	{
1028	if (rtl_ocp_reg_failure(reg))
1029	return 0;
1030
1031	RTL_W32(tp, OCPDR, reg << 15);
1032
1033	return RTL_R32(tp, OCPDR);
1034	}
1035
1036	static u16 r8168_mac_ocp_read(struct rtl8169_private *tp, u32 reg)
1037	{
1038	unsigned long flags;
1039	u16 val;
1040
1041	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
1042	val = __r8168_mac_ocp_read(tp, reg);
1043	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
1044
1045	return val;
1046	}
1047
1048	static void r8168_mac_ocp_modify(struct rtl8169_private *tp, u32 reg, u16 mask,
1049	u16 set)
1050	{
1051	unsigned long flags;
1052	u16 data;
1053
1054	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
1055	data = __r8168_mac_ocp_read(tp, reg);
1056	__r8168_mac_ocp_write(tp, reg, data: (data & ~mask) | set);
1057	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
1058	}
1059
1060	/* Work around a hw issue with RTL8168g PHY, the quirk disables
1061	* PHY MCU interrupts before PHY power-down.
1062	*/
1063	static void rtl8168g_phy_suspend_quirk(struct rtl8169_private *tp, int value)
1064	{
1065	switch (tp->mac_version) {
1066	case RTL_GIGA_MAC_VER_40:
1067	if (value & BMCR_RESET || !(value & BMCR_PDOWN))
1068	rtl_eri_set_bits(tp, addr: 0x1a8, p: 0xfc000000);
1069	else
1070	rtl_eri_clear_bits(tp, addr: 0x1a8, m: 0xfc000000);
1071	break;
1072	default:
1073	break;
1074	}
1075	};
1076
1077	static void r8168g_mdio_write(struct rtl8169_private *tp, int reg, int value)
1078	{
1079	if (reg == 0x1f) {
1080	tp->ocp_base = value ? value << 4 : OCP_STD_PHY_BASE;
1081	return;
1082	}
1083
1084	if (tp->ocp_base != OCP_STD_PHY_BASE)
1085	reg -= 0x10;
1086
1087	if (tp->ocp_base == OCP_STD_PHY_BASE && reg == MII_BMCR)
1088	rtl8168g_phy_suspend_quirk(tp, value);
1089
1090	r8168_phy_ocp_write(tp, reg: tp->ocp_base + reg * 2, data: value);
1091	}
1092
1093	static int r8168g_mdio_read(struct rtl8169_private *tp, int reg)
1094	{
1095	if (reg == 0x1f)
1096	return tp->ocp_base == OCP_STD_PHY_BASE ? 0 : tp->ocp_base >> 4;
1097
1098	if (tp->ocp_base != OCP_STD_PHY_BASE)
1099	reg -= 0x10;
1100
1101	return r8168_phy_ocp_read(tp, reg: tp->ocp_base + reg * 2);
1102	}
1103
1104	static void mac_mcu_write(struct rtl8169_private *tp, int reg, int value)
1105	{
1106	if (reg == 0x1f) {
1107	tp->ocp_base = value << 4;
1108	return;
1109	}
1110
1111	r8168_mac_ocp_write(tp, reg: tp->ocp_base + reg, data: value);
1112	}
1113
1114	static int mac_mcu_read(struct rtl8169_private *tp, int reg)
1115	{
1116	return r8168_mac_ocp_read(tp, reg: tp->ocp_base + reg);
1117	}
1118
1119	DECLARE_RTL_COND(rtl_phyar_cond)
1120	{
1121	return RTL_R32(tp, PHYAR) & 0x80000000;
1122	}
1123
1124	static void r8169_mdio_write(struct rtl8169_private *tp, int reg, int value)
1125	{
1126	RTL_W32(tp, PHYAR, 0x80000000 | (reg & 0x1f) << 16 | (value & 0xffff));
1127
1128	rtl_loop_wait_low(tp, c: &rtl_phyar_cond, d: 25, n: 20);
1129	/*
1130	* According to hardware specs a 20us delay is required after write
1131	* complete indication, but before sending next command.
1132	*/
1133	udelay(20);
1134	}
1135
1136	static int r8169_mdio_read(struct rtl8169_private *tp, int reg)
1137	{
1138	int value;
1139
1140	RTL_W32(tp, PHYAR, 0x0 | (reg & 0x1f) << 16);
1141
1142	value = rtl_loop_wait_high(tp, c: &rtl_phyar_cond, d: 25, n: 20) ?
1143	RTL_R32(tp, PHYAR) & 0xffff : -ETIMEDOUT;
1144
1145	/*
1146	* According to hardware specs a 20us delay is required after read
1147	* complete indication, but before sending next command.
1148	*/
1149	udelay(20);
1150
1151	return value;
1152	}
1153
1154	DECLARE_RTL_COND(rtl_ocpar_cond)
1155	{
1156	return RTL_R32(tp, OCPAR) & OCPAR_FLAG;
1157	}
1158
1159	#define R8168DP_1_MDIO_ACCESS_BIT 0x00020000
1160
1161	static void r8168dp_2_mdio_start(struct rtl8169_private *tp)
1162	{
1163	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) & ~R8168DP_1_MDIO_ACCESS_BIT);
1164	}
1165
1166	static void r8168dp_2_mdio_stop(struct rtl8169_private *tp)
1167	{
1168	RTL_W32(tp, 0xd0, RTL_R32(tp, 0xd0) | R8168DP_1_MDIO_ACCESS_BIT);
1169	}
1170
1171	static void r8168dp_2_mdio_write(struct rtl8169_private *tp, int reg, int value)
1172	{
1173	r8168dp_2_mdio_start(tp);
1174
1175	r8169_mdio_write(tp, reg, value);
1176
1177	r8168dp_2_mdio_stop(tp);
1178	}
1179
1180	static int r8168dp_2_mdio_read(struct rtl8169_private *tp, int reg)
1181	{
1182	int value;
1183
1184	/* Work around issue with chip reporting wrong PHY ID */
1185	if (reg == MII_PHYSID2)
1186	return 0xc912;
1187
1188	r8168dp_2_mdio_start(tp);
1189
1190	value = r8169_mdio_read(tp, reg);
1191
1192	r8168dp_2_mdio_stop(tp);
1193
1194	return value;
1195	}
1196
1197	static void rtl_writephy(struct rtl8169_private *tp, int location, int val)
1198	{
1199	switch (tp->mac_version) {
1200	case RTL_GIGA_MAC_VER_28:
1201	case RTL_GIGA_MAC_VER_31:
1202	r8168dp_2_mdio_write(tp, reg: location, value: val);
1203	break;
1204	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_65:
1205	r8168g_mdio_write(tp, reg: location, value: val);
1206	break;
1207	default:
1208	r8169_mdio_write(tp, reg: location, value: val);
1209	break;
1210	}
1211	}
1212
1213	static int rtl_readphy(struct rtl8169_private *tp, int location)
1214	{
1215	switch (tp->mac_version) {
1216	case RTL_GIGA_MAC_VER_28:
1217	case RTL_GIGA_MAC_VER_31:
1218	return r8168dp_2_mdio_read(tp, reg: location);
1219	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_65:
1220	return r8168g_mdio_read(tp, reg: location);
1221	default:
1222	return r8169_mdio_read(tp, reg: location);
1223	}
1224	}
1225
1226	DECLARE_RTL_COND(rtl_ephyar_cond)
1227	{
1228	return RTL_R32(tp, EPHYAR) & EPHYAR_FLAG;
1229	}
1230
1231	static void rtl_ephy_write(struct rtl8169_private *tp, int reg_addr, int value)
1232	{
1233	RTL_W32(tp, EPHYAR, EPHYAR_WRITE_CMD | (value & EPHYAR_DATA_MASK) |
1234	(reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1235
1236	rtl_loop_wait_low(tp, c: &rtl_ephyar_cond, d: 10, n: 100);
1237
1238	udelay(10);
1239	}
1240
1241	static u16 rtl_ephy_read(struct rtl8169_private *tp, int reg_addr)
1242	{
1243	RTL_W32(tp, EPHYAR, (reg_addr & EPHYAR_REG_MASK) << EPHYAR_REG_SHIFT);
1244
1245	return rtl_loop_wait_high(tp, c: &rtl_ephyar_cond, d: 10, n: 100) ?
1246	RTL_R32(tp, EPHYAR) & EPHYAR_DATA_MASK : ~0;
1247	}
1248
1249	static u32 r8168dp_ocp_read(struct rtl8169_private *tp, u16 reg)
1250	{
1251	RTL_W32(tp, OCPAR, 0x0fu << 12 | (reg & 0x0fff));
1252	return rtl_loop_wait_high(tp, c: &rtl_ocpar_cond, d: 100, n: 20) ?
1253	RTL_R32(tp, OCPDR) : ~0;
1254	}
1255
1256	static u32 r8168ep_ocp_read(struct rtl8169_private *tp, u16 reg)
1257	{
1258	return _rtl_eri_read(tp, addr: reg, ERIAR_OOB);
1259	}
1260
1261	static void r8168dp_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1262	u32 data)
1263	{
1264	RTL_W32(tp, OCPDR, data);
1265	RTL_W32(tp, OCPAR, OCPAR_FLAG | ((u32)mask & 0x0f) << 12 | (reg & 0x0fff));
1266	rtl_loop_wait_low(tp, c: &rtl_ocpar_cond, d: 100, n: 20);
1267	}
1268
1269	static void r8168ep_ocp_write(struct rtl8169_private *tp, u8 mask, u16 reg,
1270	u32 data)
1271	{
1272	_rtl_eri_write(tp, addr: reg, mask: ((u32)mask & 0x0f) << ERIAR_MASK_SHIFT,
1273	val: data, ERIAR_OOB);
1274	}
1275
1276	static void r8168dp_oob_notify(struct rtl8169_private *tp, u8 cmd)
1277	{
1278	rtl_eri_write(tp, addr: 0xe8, ERIAR_MASK_0001, val: cmd);
1279
1280	r8168dp_ocp_write(tp, mask: 0x1, reg: 0x30, data: 0x00000001);
1281	}
1282
1283	#define OOB_CMD_RESET 0x00
1284	#define OOB_CMD_DRIVER_START 0x05
1285	#define OOB_CMD_DRIVER_STOP 0x06
1286
1287	static u16 rtl8168_get_ocp_reg(struct rtl8169_private *tp)
1288	{
1289	return (tp->mac_version == RTL_GIGA_MAC_VER_31) ? 0xb8 : 0x10;
1290	}
1291
1292	DECLARE_RTL_COND(rtl_dp_ocp_read_cond)
1293	{
1294	u16 reg;
1295
1296	reg = rtl8168_get_ocp_reg(tp);
1297
1298	return r8168dp_ocp_read(tp, reg) & 0x00000800;
1299	}
1300
1301	DECLARE_RTL_COND(rtl_ep_ocp_read_cond)
1302	{
1303	return r8168ep_ocp_read(tp, reg: 0x124) & 0x00000001;
1304	}
1305
1306	DECLARE_RTL_COND(rtl_ocp_tx_cond)
1307	{
1308	return RTL_R8(tp, IBISR0) & 0x20;
1309	}
1310
1311	static void rtl8168ep_stop_cmac(struct rtl8169_private *tp)
1312	{
1313	RTL_W8(tp, IBCR2, RTL_R8(tp, IBCR2) & ~0x01);
1314	rtl_loop_wait_high(tp, c: &rtl_ocp_tx_cond, d: 50000, n: 2000);
1315	RTL_W8(tp, IBISR0, RTL_R8(tp, IBISR0) | 0x20);
1316	RTL_W8(tp, IBCR0, RTL_R8(tp, IBCR0) & ~0x01);
1317	}
1318
1319	static void rtl_dash_loop_wait(struct rtl8169_private *tp,
1320	const struct rtl_cond *c,
1321	unsigned long usecs, int n, bool high)
1322	{
1323	if (!tp->dash_enabled)
1324	return;
1325	rtl_loop_wait(tp, c, usecs, n, high);
1326	}
1327
1328	static void rtl_dash_loop_wait_high(struct rtl8169_private *tp,
1329	const struct rtl_cond *c,
1330	unsigned long d, int n)
1331	{
1332	rtl_dash_loop_wait(tp, c, usecs: d, n, high: true);
1333	}
1334
1335	static void rtl_dash_loop_wait_low(struct rtl8169_private *tp,
1336	const struct rtl_cond *c,
1337	unsigned long d, int n)
1338	{
1339	rtl_dash_loop_wait(tp, c, usecs: d, n, high: false);
1340	}
1341
1342	static void rtl8168dp_driver_start(struct rtl8169_private *tp)
1343	{
1344	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_START);
1345	rtl_dash_loop_wait_high(tp, c: &rtl_dp_ocp_read_cond, d: 10000, n: 10);
1346	}
1347
1348	static void rtl8168ep_driver_start(struct rtl8169_private *tp)
1349	{
1350	r8168ep_ocp_write(tp, mask: 0x01, reg: 0x180, OOB_CMD_DRIVER_START);
1351	r8168ep_ocp_write(tp, mask: 0x01, reg: 0x30, data: r8168ep_ocp_read(tp, reg: 0x30) | 0x01);
1352	rtl_dash_loop_wait_high(tp, c: &rtl_ep_ocp_read_cond, d: 10000, n: 30);
1353	}
1354
1355	static void rtl8168_driver_start(struct rtl8169_private *tp)
1356	{
1357	if (tp->dash_type == RTL_DASH_DP)
1358	rtl8168dp_driver_start(tp);
1359	else
1360	rtl8168ep_driver_start(tp);
1361	}
1362
1363	static void rtl8168dp_driver_stop(struct rtl8169_private *tp)
1364	{
1365	r8168dp_oob_notify(tp, OOB_CMD_DRIVER_STOP);
1366	rtl_dash_loop_wait_low(tp, c: &rtl_dp_ocp_read_cond, d: 10000, n: 10);
1367	}
1368
1369	static void rtl8168ep_driver_stop(struct rtl8169_private *tp)
1370	{
1371	rtl8168ep_stop_cmac(tp);
1372	r8168ep_ocp_write(tp, mask: 0x01, reg: 0x180, OOB_CMD_DRIVER_STOP);
1373	r8168ep_ocp_write(tp, mask: 0x01, reg: 0x30, data: r8168ep_ocp_read(tp, reg: 0x30) | 0x01);
1374	rtl_dash_loop_wait_low(tp, c: &rtl_ep_ocp_read_cond, d: 10000, n: 10);
1375	}
1376
1377	static void rtl8168_driver_stop(struct rtl8169_private *tp)
1378	{
1379	if (tp->dash_type == RTL_DASH_DP)
1380	rtl8168dp_driver_stop(tp);
1381	else
1382	rtl8168ep_driver_stop(tp);
1383	}
1384
1385	static bool r8168dp_check_dash(struct rtl8169_private *tp)
1386	{
1387	u16 reg = rtl8168_get_ocp_reg(tp);
1388
1389	return r8168dp_ocp_read(tp, reg) & BIT(15);
1390	}
1391
1392	static bool r8168ep_check_dash(struct rtl8169_private *tp)
1393	{
1394	return r8168ep_ocp_read(tp, reg: 0x128) & BIT(0);
1395	}
1396
1397	static bool rtl_dash_is_enabled(struct rtl8169_private *tp)
1398	{
1399	switch (tp->dash_type) {
1400	case RTL_DASH_DP:
1401	return r8168dp_check_dash(tp);
1402	case RTL_DASH_EP:
1403	return r8168ep_check_dash(tp);
1404	default:
1405	return false;
1406	}
1407	}
1408
1409	static enum rtl_dash_type rtl_get_dash_type(struct rtl8169_private *tp)
1410	{
1411	switch (tp->mac_version) {
1412	case RTL_GIGA_MAC_VER_28:
1413	case RTL_GIGA_MAC_VER_31:
1414	return RTL_DASH_DP;
1415	case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53:
1416	return RTL_DASH_EP;
1417	default:
1418	return RTL_DASH_NONE;
1419	}
1420	}
1421
1422	static void rtl_set_d3_pll_down(struct rtl8169_private *tp, bool enable)
1423	{
1424	switch (tp->mac_version) {
1425	case RTL_GIGA_MAC_VER_25 ... RTL_GIGA_MAC_VER_26:
1426	case RTL_GIGA_MAC_VER_29 ... RTL_GIGA_MAC_VER_30:
1427	case RTL_GIGA_MAC_VER_32 ... RTL_GIGA_MAC_VER_37:
1428	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_65:
1429	if (enable)
1430	RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) & ~D3_NO_PLL_DOWN);
1431	else
1432	RTL_W8(tp, PMCH, RTL_R8(tp, PMCH) | D3_NO_PLL_DOWN);
1433	break;
1434	default:
1435	break;
1436	}
1437	}
1438
1439	static void rtl_reset_packet_filter(struct rtl8169_private *tp)
1440	{
1441	rtl_eri_clear_bits(tp, addr: 0xdc, BIT(0));
1442	rtl_eri_set_bits(tp, addr: 0xdc, BIT(0));
1443	}
1444
1445	DECLARE_RTL_COND(rtl_efusear_cond)
1446	{
1447	return RTL_R32(tp, EFUSEAR) & EFUSEAR_FLAG;
1448	}
1449
1450	u8 rtl8168d_efuse_read(struct rtl8169_private *tp, int reg_addr)
1451	{
1452	RTL_W32(tp, EFUSEAR, (reg_addr & EFUSEAR_REG_MASK) << EFUSEAR_REG_SHIFT);
1453
1454	return rtl_loop_wait_high(tp, c: &rtl_efusear_cond, d: 100, n: 300) ?
1455	RTL_R32(tp, EFUSEAR) & EFUSEAR_DATA_MASK : ~0;
1456	}
1457
1458	static u32 rtl_get_events(struct rtl8169_private *tp)
1459	{
1460	if (rtl_is_8125(tp))
1461	return RTL_R32(tp, IntrStatus_8125);
1462	else
1463	return RTL_R16(tp, IntrStatus);
1464	}
1465
1466	static void rtl_ack_events(struct rtl8169_private *tp, u32 bits)
1467	{
1468	if (rtl_is_8125(tp))
1469	RTL_W32(tp, IntrStatus_8125, bits);
1470	else
1471	RTL_W16(tp, IntrStatus, bits);
1472	}
1473
1474	static void rtl_irq_disable(struct rtl8169_private *tp)
1475	{
1476	if (rtl_is_8125(tp))
1477	RTL_W32(tp, IntrMask_8125, 0);
1478	else
1479	RTL_W16(tp, IntrMask, 0);
1480	}
1481
1482	static void rtl_irq_enable(struct rtl8169_private *tp)
1483	{
1484	if (rtl_is_8125(tp))
1485	RTL_W32(tp, IntrMask_8125, tp->irq_mask);
1486	else
1487	RTL_W16(tp, IntrMask, tp->irq_mask);
1488	}
1489
1490	static void rtl8169_irq_mask_and_ack(struct rtl8169_private *tp)
1491	{
1492	rtl_irq_disable(tp);
1493	rtl_ack_events(tp, bits: 0xffffffff);
1494	rtl_pci_commit(tp);
1495	}
1496
1497	static void rtl_link_chg_patch(struct rtl8169_private *tp)
1498	{
1499	struct phy_device *phydev = tp->phydev;
1500
1501	if (tp->mac_version == RTL_GIGA_MAC_VER_34 ||
1502	tp->mac_version == RTL_GIGA_MAC_VER_38) {
1503	if (phydev->speed == SPEED_1000) {
1504	rtl_eri_write(tp, addr: 0x1bc, ERIAR_MASK_1111, val: 0x00000011);
1505	rtl_eri_write(tp, addr: 0x1dc, ERIAR_MASK_1111, val: 0x00000005);
1506	} else if (phydev->speed == SPEED_100) {
1507	rtl_eri_write(tp, addr: 0x1bc, ERIAR_MASK_1111, val: 0x0000001f);
1508	rtl_eri_write(tp, addr: 0x1dc, ERIAR_MASK_1111, val: 0x00000005);
1509	} else {
1510	rtl_eri_write(tp, addr: 0x1bc, ERIAR_MASK_1111, val: 0x0000001f);
1511	rtl_eri_write(tp, addr: 0x1dc, ERIAR_MASK_1111, val: 0x0000003f);
1512	}
1513	rtl_reset_packet_filter(tp);
1514	} else if (tp->mac_version == RTL_GIGA_MAC_VER_35 ||
1515	tp->mac_version == RTL_GIGA_MAC_VER_36) {
1516	if (phydev->speed == SPEED_1000) {
1517	rtl_eri_write(tp, addr: 0x1bc, ERIAR_MASK_1111, val: 0x00000011);
1518	rtl_eri_write(tp, addr: 0x1dc, ERIAR_MASK_1111, val: 0x00000005);
1519	} else {
1520	rtl_eri_write(tp, addr: 0x1bc, ERIAR_MASK_1111, val: 0x0000001f);
1521	rtl_eri_write(tp, addr: 0x1dc, ERIAR_MASK_1111, val: 0x0000003f);
1522	}
1523	} else if (tp->mac_version == RTL_GIGA_MAC_VER_37) {
1524	if (phydev->speed == SPEED_10) {
1525	rtl_eri_write(tp, addr: 0x1d0, ERIAR_MASK_0011, val: 0x4d02);
1526	rtl_eri_write(tp, addr: 0x1dc, ERIAR_MASK_0011, val: 0x0060a);
1527	} else {
1528	rtl_eri_write(tp, addr: 0x1d0, ERIAR_MASK_0011, val: 0x0000);
1529	}
1530	}
1531	}
1532
1533	#define WAKE_ANY (WAKE_PHY | WAKE_MAGIC | WAKE_UCAST | WAKE_BCAST | WAKE_MCAST)
1534
1535	static void rtl8169_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1536	{
1537	struct rtl8169_private *tp = netdev_priv(dev);
1538
1539	wol->supported = WAKE_ANY;
1540	wol->wolopts = tp->saved_wolopts;
1541	}
1542
1543	static void __rtl8169_set_wol(struct rtl8169_private *tp, u32 wolopts)
1544	{
1545	static const struct {
1546	u32 opt;
1547	u16 reg;
1548	u8 mask;
1549	} cfg[] = {
1550	{ WAKE_PHY, Config3, LinkUp },
1551	{ WAKE_UCAST, Config5, UWF },
1552	{ WAKE_BCAST, Config5, BWF },
1553	{ WAKE_MCAST, Config5, MWF },
1554	{ WAKE_ANY, Config5, LanWake },
1555	{ WAKE_MAGIC, Config3, MagicPacket }
1556	};
1557	unsigned int i, tmp = ARRAY_SIZE(cfg);
1558	unsigned long flags;
1559	u8 options;
1560
1561	rtl_unlock_config_regs(tp);
1562
1563	if (rtl_is_8168evl_up(tp)) {
1564	tmp--;
1565	if (wolopts & WAKE_MAGIC)
1566	rtl_eri_set_bits(tp, addr: 0x0dc, p: MagicPacket_v2);
1567	else
1568	rtl_eri_clear_bits(tp, addr: 0x0dc, m: MagicPacket_v2);
1569	} else if (rtl_is_8125(tp)) {
1570	tmp--;
1571	if (wolopts & WAKE_MAGIC)
1572	r8168_mac_ocp_modify(tp, reg: 0xc0b6, mask: 0, BIT(0));
1573	else
1574	r8168_mac_ocp_modify(tp, reg: 0xc0b6, BIT(0), set: 0);
1575	}
1576
1577	raw_spin_lock_irqsave(&tp->config25_lock, flags);
1578	for (i = 0; i < tmp; i++) {
1579	options = RTL_R8(tp, cfg[i].reg) & ~cfg[i].mask;
1580	if (wolopts & cfg[i].opt)
1581	options |= cfg[i].mask;
1582	RTL_W8(tp, cfg[i].reg, options);
1583	}
1584	raw_spin_unlock_irqrestore(&tp->config25_lock, flags);
1585
1586	switch (tp->mac_version) {
1587	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
1588	options = RTL_R8(tp, Config1) & ~PMEnable;
1589	if (wolopts)
1590	options |= PMEnable;
1591	RTL_W8(tp, Config1, options);
1592	break;
1593	case RTL_GIGA_MAC_VER_34:
1594	case RTL_GIGA_MAC_VER_37:
1595	case RTL_GIGA_MAC_VER_39 ... RTL_GIGA_MAC_VER_65:
1596	if (wolopts)
1597	rtl_mod_config2(tp, clear: 0, PME_SIGNAL);
1598	else
1599	rtl_mod_config2(tp, PME_SIGNAL, set: 0);
1600	break;
1601	default:
1602	break;
1603	}
1604
1605	rtl_lock_config_regs(tp);
1606
1607	device_set_wakeup_enable(dev: tp_to_dev(tp), enable: wolopts);
1608
1609	if (!tp->dash_enabled) {
1610	rtl_set_d3_pll_down(tp, enable: !wolopts);
1611	tp->dev->wol_enabled = wolopts ? 1 : 0;
1612	}
1613	}
1614
1615	static int rtl8169_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1616	{
1617	struct rtl8169_private *tp = netdev_priv(dev);
1618
1619	if (wol->wolopts & ~WAKE_ANY)
1620	return -EINVAL;
1621
1622	tp->saved_wolopts = wol->wolopts;
1623	__rtl8169_set_wol(tp, wolopts: tp->saved_wolopts);
1624
1625	return 0;
1626	}
1627
1628	static void rtl8169_get_drvinfo(struct net_device *dev,
1629	struct ethtool_drvinfo *info)
1630	{
1631	struct rtl8169_private *tp = netdev_priv(dev);
1632	struct rtl_fw *rtl_fw = tp->rtl_fw;
1633
1634	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1635	strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
1636	BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version));
1637	if (rtl_fw)
1638	strscpy(info->fw_version, rtl_fw->version,
1639	sizeof(info->fw_version));
1640	}
1641
1642	static int rtl8169_get_regs_len(struct net_device *dev)
1643	{
1644	return R8169_REGS_SIZE;
1645	}
1646
1647	static netdev_features_t rtl8169_fix_features(struct net_device *dev,
1648	netdev_features_t features)
1649	{
1650	struct rtl8169_private *tp = netdev_priv(dev);
1651
1652	if (dev->mtu > TD_MSS_MAX)
1653	features &= ~NETIF_F_ALL_TSO;
1654
1655	if (dev->mtu > ETH_DATA_LEN &&
1656	tp->mac_version > RTL_GIGA_MAC_VER_06)
1657	features &= ~(NETIF_F_CSUM_MASK | NETIF_F_ALL_TSO);
1658
1659	return features;
1660	}
1661
1662	static void rtl_set_rx_config_features(struct rtl8169_private *tp,
1663	netdev_features_t features)
1664	{
1665	u32 rx_config = RTL_R32(tp, RxConfig);
1666
1667	if (features & NETIF_F_RXALL)
1668	rx_config |= RX_CONFIG_ACCEPT_ERR_MASK;
1669	else
1670	rx_config &= ~RX_CONFIG_ACCEPT_ERR_MASK;
1671
1672	if (rtl_is_8125(tp)) {
1673	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1674	rx_config |= RX_VLAN_8125;
1675	else
1676	rx_config &= ~RX_VLAN_8125;
1677	}
1678
1679	RTL_W32(tp, RxConfig, rx_config);
1680	}
1681
1682	static int rtl8169_set_features(struct net_device *dev,
1683	netdev_features_t features)
1684	{
1685	struct rtl8169_private *tp = netdev_priv(dev);
1686
1687	rtl_set_rx_config_features(tp, features);
1688
1689	if (features & NETIF_F_RXCSUM)
1690	tp->cp_cmd |= RxChkSum;
1691	else
1692	tp->cp_cmd &= ~RxChkSum;
1693
1694	if (!rtl_is_8125(tp)) {
1695	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1696	tp->cp_cmd |= RxVlan;
1697	else
1698	tp->cp_cmd &= ~RxVlan;
1699	}
1700
1701	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
1702	rtl_pci_commit(tp);
1703
1704	return 0;
1705	}
1706
1707	static inline u32 rtl8169_tx_vlan_tag(struct sk_buff *skb)
1708	{
1709	return (skb_vlan_tag_present(skb)) ?
1710	TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00;
1711	}
1712
1713	static void rtl8169_rx_vlan_tag(struct RxDesc *desc, struct sk_buff *skb)
1714	{
1715	u32 opts2 = le32_to_cpu(desc->opts2);
1716
1717	if (opts2 & RxVlanTag)
1718	__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff));
1719	}
1720
1721	static void rtl8169_get_regs(struct net_device *dev, struct ethtool_regs *regs,
1722	void *p)
1723	{
1724	struct rtl8169_private *tp = netdev_priv(dev);
1725	u32 __iomem *data = tp->mmio_addr;
1726	u32 *dw = p;
1727	int i;
1728
1729	for (i = 0; i < R8169_REGS_SIZE; i += 4)
1730	memcpy_fromio(dw++, data++, 4);
1731	}
1732
1733	static const char rtl8169_gstrings[][ETH_GSTRING_LEN] = {
1734	"tx_packets",
1735	"rx_packets",
1736	"tx_errors",
1737	"rx_errors",
1738	"rx_missed",
1739	"align_errors",
1740	"tx_single_collisions",
1741	"tx_multi_collisions",
1742	"unicast",
1743	"broadcast",
1744	"multicast",
1745	"tx_aborted",
1746	"tx_underrun",
1747	};
1748
1749	static int rtl8169_get_sset_count(struct net_device *dev, int sset)
1750	{
1751	switch (sset) {
1752	case ETH_SS_STATS:
1753	return ARRAY_SIZE(rtl8169_gstrings);
1754	default:
1755	return -EOPNOTSUPP;
1756	}
1757	}
1758
1759	DECLARE_RTL_COND(rtl_counters_cond)
1760	{
1761	return RTL_R32(tp, CounterAddrLow) & (CounterReset | CounterDump);
1762	}
1763
1764	static void rtl8169_do_counters(struct rtl8169_private *tp, u32 counter_cmd)
1765	{
1766	u32 cmd = lower_32_bits(tp->counters_phys_addr);
1767
1768	RTL_W32(tp, CounterAddrHigh, upper_32_bits(tp->counters_phys_addr));
1769	rtl_pci_commit(tp);
1770	RTL_W32(tp, CounterAddrLow, cmd);
1771	RTL_W32(tp, CounterAddrLow, cmd | counter_cmd);
1772
1773	rtl_loop_wait_low(tp, c: &rtl_counters_cond, d: 10, n: 1000);
1774	}
1775
1776	static void rtl8169_update_counters(struct rtl8169_private *tp)
1777	{
1778	u8 val = RTL_R8(tp, ChipCmd);
1779
1780	/*
1781	* Some chips are unable to dump tally counters when the receiver
1782	* is disabled. If 0xff chip may be in a PCI power-save state.
1783	*/
1784	if (val & CmdRxEnb && val != 0xff)
1785	rtl8169_do_counters(tp, counter_cmd: CounterDump);
1786	}
1787
1788	static void rtl8169_init_counter_offsets(struct rtl8169_private *tp)
1789	{
1790	struct rtl8169_counters *counters = tp->counters;
1791
1792	/*
1793	* rtl8169_init_counter_offsets is called from rtl_open. On chip
1794	* versions prior to RTL_GIGA_MAC_VER_19 the tally counters are only
1795	* reset by a power cycle, while the counter values collected by the
1796	* driver are reset at every driver unload/load cycle.
1797	*
1798	* To make sure the HW values returned by @get_stats64 match the SW
1799	* values, we collect the initial values at first open(*) and use them
1800	* as offsets to normalize the values returned by @get_stats64.
1801	*
1802	* (*) We can't call rtl8169_init_counter_offsets from rtl_init_one
1803	* for the reason stated in rtl8169_update_counters; CmdRxEnb is only
1804	* set at open time by rtl_hw_start.
1805	*/
1806
1807	if (tp->tc_offset.inited)
1808	return;
1809
1810	if (tp->mac_version >= RTL_GIGA_MAC_VER_19) {
1811	rtl8169_do_counters(tp, counter_cmd: CounterReset);
1812	} else {
1813	rtl8169_update_counters(tp);
1814	tp->tc_offset.tx_errors = counters->tx_errors;
1815	tp->tc_offset.tx_multi_collision = counters->tx_multi_collision;
1816	tp->tc_offset.tx_aborted = counters->tx_aborted;
1817	tp->tc_offset.rx_missed = counters->rx_missed;
1818	}
1819
1820	tp->tc_offset.inited = true;
1821	}
1822
1823	static void rtl8169_get_ethtool_stats(struct net_device *dev,
1824	struct ethtool_stats *stats, u64 *data)
1825	{
1826	struct rtl8169_private *tp = netdev_priv(dev);
1827	struct rtl8169_counters *counters;
1828
1829	counters = tp->counters;
1830	rtl8169_update_counters(tp);
1831
1832	data[0] = le64_to_cpu(counters->tx_packets);
1833	data[1] = le64_to_cpu(counters->rx_packets);
1834	data[2] = le64_to_cpu(counters->tx_errors);
1835	data[3] = le32_to_cpu(counters->rx_errors);
1836	data[4] = le16_to_cpu(counters->rx_missed);
1837	data[5] = le16_to_cpu(counters->align_errors);
1838	data[6] = le32_to_cpu(counters->tx_one_collision);
1839	data[7] = le32_to_cpu(counters->tx_multi_collision);
1840	data[8] = le64_to_cpu(counters->rx_unicast);
1841	data[9] = le64_to_cpu(counters->rx_broadcast);
1842	data[10] = le32_to_cpu(counters->rx_multicast);
1843	data[11] = le16_to_cpu(counters->tx_aborted);
1844	data[12] = le16_to_cpu(counters->tx_underun);
1845	}
1846
1847	static void rtl8169_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1848	{
1849	switch(stringset) {
1850	case ETH_SS_STATS:
1851	memcpy(data, rtl8169_gstrings, sizeof(rtl8169_gstrings));
1852	break;
1853	}
1854	}
1855
1856	/*
1857	* Interrupt coalescing
1858	*
1859	* > 1 - the availability of the IntrMitigate (0xe2) register through the
1860	* > 8169, 8168 and 810x line of chipsets
1861	*
1862	* 8169, 8168, and 8136(810x) serial chipsets support it.
1863	*
1864	* > 2 - the Tx timer unit at gigabit speed
1865	*
1866	* The unit of the timer depends on both the speed and the setting of CPlusCmd
1867	* (0xe0) bit 1 and bit 0.
1868	*
1869	* For 8169
1870	* bit[1:0] \ speed 1000M 100M 10M
1871	* 0 0 320ns 2.56us 40.96us
1872	* 0 1 2.56us 20.48us 327.7us
1873	* 1 0 5.12us 40.96us 655.4us
1874	* 1 1 10.24us 81.92us 1.31ms
1875	*
1876	* For the other
1877	* bit[1:0] \ speed 1000M 100M 10M
1878	* 0 0 5us 2.56us 40.96us
1879	* 0 1 40us 20.48us 327.7us
1880	* 1 0 80us 40.96us 655.4us
1881	* 1 1 160us 81.92us 1.31ms
1882	*/
1883
1884	/* rx/tx scale factors for all CPlusCmd[0:1] cases */
1885	struct rtl_coalesce_info {
1886	u32 speed;
1887	u32 scale_nsecs[4];
1888	};
1889
1890	/* produce array with base delay *1, *8, *8*2, *8*2*2 */
1891	#define COALESCE_DELAY(d) { (d), 8 * (d), 16 * (d), 32 * (d) }
1892
1893	static const struct rtl_coalesce_info rtl_coalesce_info_8169[] = {
1894	{ SPEED_1000, COALESCE_DELAY(320) },
1895	{ SPEED_100, COALESCE_DELAY(2560) },
1896	{ SPEED_10, COALESCE_DELAY(40960) },
1897	{ 0 },
1898	};
1899
1900	static const struct rtl_coalesce_info rtl_coalesce_info_8168_8136[] = {
1901	{ SPEED_1000, COALESCE_DELAY(5000) },
1902	{ SPEED_100, COALESCE_DELAY(2560) },
1903	{ SPEED_10, COALESCE_DELAY(40960) },
1904	{ 0 },
1905	};
1906	#undef COALESCE_DELAY
1907
1908	/* get rx/tx scale vector corresponding to current speed */
1909	static const struct rtl_coalesce_info *
1910	rtl_coalesce_info(struct rtl8169_private *tp)
1911	{
1912	const struct rtl_coalesce_info *ci;
1913
1914	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
1915	ci = rtl_coalesce_info_8169;
1916	else
1917	ci = rtl_coalesce_info_8168_8136;
1918
1919	/* if speed is unknown assume highest one */
1920	if (tp->phydev->speed == SPEED_UNKNOWN)
1921	return ci;
1922
1923	for (; ci->speed; ci++) {
1924	if (tp->phydev->speed == ci->speed)
1925	return ci;
1926	}
1927
1928	return ERR_PTR(error: -ELNRNG);
1929	}
1930
1931	static int rtl_get_coalesce(struct net_device *dev,
1932	struct ethtool_coalesce *ec,
1933	struct kernel_ethtool_coalesce *kernel_coal,
1934	struct netlink_ext_ack *extack)
1935	{
1936	struct rtl8169_private *tp = netdev_priv(dev);
1937	const struct rtl_coalesce_info *ci;
1938	u32 scale, c_us, c_fr;
1939	u16 intrmit;
1940
1941	if (rtl_is_8125(tp))
1942	return -EOPNOTSUPP;
1943
1944	memset(ec, 0, sizeof(*ec));
1945
1946	/* get rx/tx scale corresponding to current speed and CPlusCmd[0:1] */
1947	ci = rtl_coalesce_info(tp);
1948	if (IS_ERR(ptr: ci))
1949	return PTR_ERR(ptr: ci);
1950
1951	scale = ci->scale_nsecs[tp->cp_cmd & INTT_MASK];
1952
1953	intrmit = RTL_R16(tp, IntrMitigate);
1954
1955	c_us = FIELD_GET(RTL_COALESCE_TX_USECS, intrmit);
1956	ec->tx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1957
1958	c_fr = FIELD_GET(RTL_COALESCE_TX_FRAMES, intrmit);
1959	/* ethtool_coalesce states usecs and max_frames must not both be 0 */
1960	ec->tx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1961
1962	c_us = FIELD_GET(RTL_COALESCE_RX_USECS, intrmit);
1963	ec->rx_coalesce_usecs = DIV_ROUND_UP(c_us * scale, 1000);
1964
1965	c_fr = FIELD_GET(RTL_COALESCE_RX_FRAMES, intrmit);
1966	ec->rx_max_coalesced_frames = (c_us || c_fr) ? c_fr * 4 : 1;
1967
1968	return 0;
1969	}
1970
1971	/* choose appropriate scale factor and CPlusCmd[0:1] for (speed, usec) */
1972	static int rtl_coalesce_choose_scale(struct rtl8169_private *tp, u32 usec,
1973	u16 *cp01)
1974	{
1975	const struct rtl_coalesce_info *ci;
1976	u16 i;
1977
1978	ci = rtl_coalesce_info(tp);
1979	if (IS_ERR(ptr: ci))
1980	return PTR_ERR(ptr: ci);
1981
1982	for (i = 0; i < 4; i++) {
1983	if (usec <= ci->scale_nsecs[i] * RTL_COALESCE_T_MAX / 1000U) {
1984	*cp01 = i;
1985	return ci->scale_nsecs[i];
1986	}
1987	}
1988
1989	return -ERANGE;
1990	}
1991
1992	static int rtl_set_coalesce(struct net_device *dev,
1993	struct ethtool_coalesce *ec,
1994	struct kernel_ethtool_coalesce *kernel_coal,
1995	struct netlink_ext_ack *extack)
1996	{
1997	struct rtl8169_private *tp = netdev_priv(dev);
1998	u32 tx_fr = ec->tx_max_coalesced_frames;
1999	u32 rx_fr = ec->rx_max_coalesced_frames;
2000	u32 coal_usec_max, units;
2001	u16 w = 0, cp01 = 0;
2002	int scale;
2003
2004	if (rtl_is_8125(tp))
2005	return -EOPNOTSUPP;
2006
2007	if (rx_fr > RTL_COALESCE_FRAME_MAX || tx_fr > RTL_COALESCE_FRAME_MAX)
2008	return -ERANGE;
2009
2010	coal_usec_max = max(ec->rx_coalesce_usecs, ec->tx_coalesce_usecs);
2011	scale = rtl_coalesce_choose_scale(tp, usec: coal_usec_max, cp01: &cp01);
2012	if (scale < 0)
2013	return scale;
2014
2015	/* Accept max_frames=1 we returned in rtl_get_coalesce. Accept it
2016	* not only when usecs=0 because of e.g. the following scenario:
2017	*
2018	* - both rx_usecs=0 & rx_frames=0 in hardware (no delay on RX)
2019	* - rtl_get_coalesce returns rx_usecs=0, rx_frames=1
2020	* - then user does `ethtool -C eth0 rx-usecs 100`
2021	*
2022	* Since ethtool sends to kernel whole ethtool_coalesce settings,
2023	* if we want to ignore rx_frames then it has to be set to 0.
2024	*/
2025	if (rx_fr == 1)
2026	rx_fr = 0;
2027	if (tx_fr == 1)
2028	tx_fr = 0;
2029
2030	/* HW requires time limit to be set if frame limit is set */
2031	if ((tx_fr && !ec->tx_coalesce_usecs) ||
2032	(rx_fr && !ec->rx_coalesce_usecs))
2033	return -EINVAL;
2034
2035	w |= FIELD_PREP(RTL_COALESCE_TX_FRAMES, DIV_ROUND_UP(tx_fr, 4));
2036	w |= FIELD_PREP(RTL_COALESCE_RX_FRAMES, DIV_ROUND_UP(rx_fr, 4));
2037
2038	units = DIV_ROUND_UP(ec->tx_coalesce_usecs * 1000U, scale);
2039	w |= FIELD_PREP(RTL_COALESCE_TX_USECS, units);
2040	units = DIV_ROUND_UP(ec->rx_coalesce_usecs * 1000U, scale);
2041	w |= FIELD_PREP(RTL_COALESCE_RX_USECS, units);
2042
2043	RTL_W16(tp, IntrMitigate, w);
2044
2045	/* Meaning of PktCntrDisable bit changed from RTL8168e-vl */
2046	if (rtl_is_8168evl_up(tp)) {
2047	if (!rx_fr && !tx_fr)
2048	/* disable packet counter */
2049	tp->cp_cmd |= PktCntrDisable;
2050	else
2051	tp->cp_cmd &= ~PktCntrDisable;
2052	}
2053
2054	tp->cp_cmd = (tp->cp_cmd & ~INTT_MASK) | cp01;
2055	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
2056	rtl_pci_commit(tp);
2057
2058	return 0;
2059	}
2060
2061	static void rtl_set_eee_txidle_timer(struct rtl8169_private *tp)
2062	{
2063	unsigned int timer_val = READ_ONCE(tp->dev->mtu) + ETH_HLEN + 0x20;
2064
2065	switch (tp->mac_version) {
2066	case RTL_GIGA_MAC_VER_46:
2067	case RTL_GIGA_MAC_VER_48:
2068	tp->tx_lpi_timer = timer_val;
2069	r8168_mac_ocp_write(tp, reg: 0xe048, data: timer_val);
2070	break;
2071	case RTL_GIGA_MAC_VER_61:
2072	case RTL_GIGA_MAC_VER_63:
2073	case RTL_GIGA_MAC_VER_65:
2074	tp->tx_lpi_timer = timer_val;
2075	RTL_W16(tp, EEE_TXIDLE_TIMER_8125, timer_val);
2076	break;
2077	default:
2078	break;
2079	}
2080	}
2081
2082	static unsigned int r8169_get_tx_lpi_timer_us(struct rtl8169_private *tp)
2083	{
2084	unsigned int speed = tp->phydev->speed;
2085	unsigned int timer = tp->tx_lpi_timer;
2086
2087	if (!timer || speed == SPEED_UNKNOWN)
2088	return 0;
2089
2090	/* tx_lpi_timer value is in bytes */
2091	return DIV_ROUND_CLOSEST(timer * BITS_PER_BYTE, speed);
2092	}
2093
2094	static int rtl8169_get_eee(struct net_device *dev, struct ethtool_keee *data)
2095	{
2096	struct rtl8169_private *tp = netdev_priv(dev);
2097	int ret;
2098
2099	if (!rtl_supports_eee(tp))
2100	return -EOPNOTSUPP;
2101
2102	ret = phy_ethtool_get_eee(phydev: tp->phydev, data);
2103	if (ret)
2104	return ret;
2105
2106	data->tx_lpi_timer = r8169_get_tx_lpi_timer_us(tp);
2107
2108	return 0;
2109	}
2110
2111	static int rtl8169_set_eee(struct net_device *dev, struct ethtool_keee *data)
2112	{
2113	struct rtl8169_private *tp = netdev_priv(dev);
2114
2115	if (!rtl_supports_eee(tp))
2116	return -EOPNOTSUPP;
2117
2118	return phy_ethtool_set_eee(phydev: tp->phydev, data);
2119	}
2120
2121	static void rtl8169_get_ringparam(struct net_device *dev,
2122	struct ethtool_ringparam *data,
2123	struct kernel_ethtool_ringparam *kernel_data,
2124	struct netlink_ext_ack *extack)
2125	{
2126	data->rx_max_pending = NUM_RX_DESC;
2127	data->rx_pending = NUM_RX_DESC;
2128	data->tx_max_pending = NUM_TX_DESC;
2129	data->tx_pending = NUM_TX_DESC;
2130	}
2131
2132	static void rtl8169_get_pauseparam(struct net_device *dev,
2133	struct ethtool_pauseparam *data)
2134	{
2135	struct rtl8169_private *tp = netdev_priv(dev);
2136	bool tx_pause, rx_pause;
2137
2138	phy_get_pause(phydev: tp->phydev, tx_pause: &tx_pause, rx_pause: &rx_pause);
2139
2140	data->autoneg = tp->phydev->autoneg;
2141	data->tx_pause = tx_pause ? 1 : 0;
2142	data->rx_pause = rx_pause ? 1 : 0;
2143	}
2144
2145	static int rtl8169_set_pauseparam(struct net_device *dev,
2146	struct ethtool_pauseparam *data)
2147	{
2148	struct rtl8169_private *tp = netdev_priv(dev);
2149
2150	if (dev->mtu > ETH_DATA_LEN)
2151	return -EOPNOTSUPP;
2152
2153	phy_set_asym_pause(phydev: tp->phydev, rx: data->rx_pause, tx: data->tx_pause);
2154
2155	return 0;
2156	}
2157
2158	static const struct ethtool_ops rtl8169_ethtool_ops = {
2159	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
2160	ETHTOOL_COALESCE_MAX_FRAMES,
2161	.get_drvinfo = rtl8169_get_drvinfo,
2162	.get_regs_len = rtl8169_get_regs_len,
2163	.get_link = ethtool_op_get_link,
2164	.get_coalesce = rtl_get_coalesce,
2165	.set_coalesce = rtl_set_coalesce,
2166	.get_regs = rtl8169_get_regs,
2167	.get_wol = rtl8169_get_wol,
2168	.set_wol = rtl8169_set_wol,
2169	.get_strings = rtl8169_get_strings,
2170	.get_sset_count = rtl8169_get_sset_count,
2171	.get_ethtool_stats = rtl8169_get_ethtool_stats,
2172	.get_ts_info = ethtool_op_get_ts_info,
2173	.nway_reset = phy_ethtool_nway_reset,
2174	.get_eee = rtl8169_get_eee,
2175	.set_eee = rtl8169_set_eee,
2176	.get_link_ksettings = phy_ethtool_get_link_ksettings,
2177	.set_link_ksettings = phy_ethtool_set_link_ksettings,
2178	.get_ringparam = rtl8169_get_ringparam,
2179	.get_pauseparam = rtl8169_get_pauseparam,
2180	.set_pauseparam = rtl8169_set_pauseparam,
2181	};
2182
2183	static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
2184	{
2185	/*
2186	* The driver currently handles the 8168Bf and the 8168Be identically
2187	* but they can be identified more specifically through the test below
2188	* if needed:
2189	*
2190	* (RTL_R32(tp, TxConfig) & 0x700000) == 0x500000 ? 8168Bf : 8168Be
2191	*
2192	* Same thing for the 8101Eb and the 8101Ec:
2193	*
2194	* (RTL_R32(tp, TxConfig) & 0x700000) == 0x200000 ? 8101Eb : 8101Ec
2195	*/
2196	static const struct rtl_mac_info {
2197	u16 mask;
2198	u16 val;
2199	enum mac_version ver;
2200	} mac_info[] = {
2201	/* 8126A family. */
2202	{ 0x7cf, 0x649, RTL_GIGA_MAC_VER_65 },
2203
2204	/* 8125B family. */
2205	{ 0x7cf, 0x641, RTL_GIGA_MAC_VER_63 },
2206
2207	/* 8125A family. */
2208	{ 0x7cf, 0x609, RTL_GIGA_MAC_VER_61 },
2209	/* It seems only XID 609 made it to the mass market.
2210	* { 0x7cf, 0x608, RTL_GIGA_MAC_VER_60 },
2211	* { 0x7c8, 0x608, RTL_GIGA_MAC_VER_61 },
2212	*/
2213
2214	/* RTL8117 */
2215	{ 0x7cf, 0x54b, RTL_GIGA_MAC_VER_53 },
2216	{ 0x7cf, 0x54a, RTL_GIGA_MAC_VER_52 },
2217
2218	/* 8168EP family. */
2219	{ 0x7cf, 0x502, RTL_GIGA_MAC_VER_51 },
2220	/* It seems this chip version never made it to
2221	* the wild. Let's disable detection.
2222	* { 0x7cf, 0x501, RTL_GIGA_MAC_VER_50 },
2223	* { 0x7cf, 0x500, RTL_GIGA_MAC_VER_49 },
2224	*/
2225
2226	/* 8168H family. */
2227	{ 0x7cf, 0x541, RTL_GIGA_MAC_VER_46 },
2228	/* It seems this chip version never made it to
2229	* the wild. Let's disable detection.
2230	* { 0x7cf, 0x540, RTL_GIGA_MAC_VER_45 },
2231	*/
2232
2233	/* 8168G family. */
2234	{ 0x7cf, 0x5c8, RTL_GIGA_MAC_VER_44 },
2235	{ 0x7cf, 0x509, RTL_GIGA_MAC_VER_42 },
2236	/* It seems this chip version never made it to
2237	* the wild. Let's disable detection.
2238	* { 0x7cf, 0x4c1, RTL_GIGA_MAC_VER_41 },
2239	*/
2240	{ 0x7cf, 0x4c0, RTL_GIGA_MAC_VER_40 },
2241
2242	/* 8168F family. */
2243	{ 0x7c8, 0x488, RTL_GIGA_MAC_VER_38 },
2244	{ 0x7cf, 0x481, RTL_GIGA_MAC_VER_36 },
2245	{ 0x7cf, 0x480, RTL_GIGA_MAC_VER_35 },
2246
2247	/* 8168E family. */
2248	{ 0x7c8, 0x2c8, RTL_GIGA_MAC_VER_34 },
2249	{ 0x7cf, 0x2c1, RTL_GIGA_MAC_VER_32 },
2250	{ 0x7c8, 0x2c0, RTL_GIGA_MAC_VER_33 },
2251
2252	/* 8168D family. */
2253	{ 0x7cf, 0x281, RTL_GIGA_MAC_VER_25 },
2254	{ 0x7c8, 0x280, RTL_GIGA_MAC_VER_26 },
2255
2256	/* 8168DP family. */
2257	/* It seems this early RTL8168dp version never made it to
2258	* the wild. Support has been removed.
2259	* { 0x7cf, 0x288, RTL_GIGA_MAC_VER_27 },
2260	*/
2261	{ 0x7cf, 0x28a, RTL_GIGA_MAC_VER_28 },
2262	{ 0x7cf, 0x28b, RTL_GIGA_MAC_VER_31 },
2263
2264	/* 8168C family. */
2265	{ 0x7cf, 0x3c9, RTL_GIGA_MAC_VER_23 },
2266	{ 0x7cf, 0x3c8, RTL_GIGA_MAC_VER_18 },
2267	{ 0x7c8, 0x3c8, RTL_GIGA_MAC_VER_24 },
2268	{ 0x7cf, 0x3c0, RTL_GIGA_MAC_VER_19 },
2269	{ 0x7cf, 0x3c2, RTL_GIGA_MAC_VER_20 },
2270	{ 0x7cf, 0x3c3, RTL_GIGA_MAC_VER_21 },
2271	{ 0x7c8, 0x3c0, RTL_GIGA_MAC_VER_22 },
2272
2273	/* 8168B family. */
2274	{ 0x7c8, 0x380, RTL_GIGA_MAC_VER_17 },
2275	{ 0x7c8, 0x300, RTL_GIGA_MAC_VER_11 },
2276
2277	/* 8101 family. */
2278	{ 0x7c8, 0x448, RTL_GIGA_MAC_VER_39 },
2279	{ 0x7c8, 0x440, RTL_GIGA_MAC_VER_37 },
2280	{ 0x7cf, 0x409, RTL_GIGA_MAC_VER_29 },
2281	{ 0x7c8, 0x408, RTL_GIGA_MAC_VER_30 },
2282	{ 0x7cf, 0x349, RTL_GIGA_MAC_VER_08 },
2283	{ 0x7cf, 0x249, RTL_GIGA_MAC_VER_08 },
2284	{ 0x7cf, 0x348, RTL_GIGA_MAC_VER_07 },
2285	{ 0x7cf, 0x248, RTL_GIGA_MAC_VER_07 },
2286	{ 0x7cf, 0x240, RTL_GIGA_MAC_VER_14 },
2287	{ 0x7c8, 0x348, RTL_GIGA_MAC_VER_09 },
2288	{ 0x7c8, 0x248, RTL_GIGA_MAC_VER_09 },
2289	{ 0x7c8, 0x340, RTL_GIGA_MAC_VER_10 },
2290
2291	/* 8110 family. */
2292	{ 0xfc8, 0x980, RTL_GIGA_MAC_VER_06 },
2293	{ 0xfc8, 0x180, RTL_GIGA_MAC_VER_05 },
2294	{ 0xfc8, 0x100, RTL_GIGA_MAC_VER_04 },
2295	{ 0xfc8, 0x040, RTL_GIGA_MAC_VER_03 },
2296	{ 0xfc8, 0x008, RTL_GIGA_MAC_VER_02 },
2297
2298	/* Catch-all */
2299	{ 0x000, 0x000, RTL_GIGA_MAC_NONE }
2300	};
2301	const struct rtl_mac_info *p = mac_info;
2302	enum mac_version ver;
2303
2304	while ((xid & p->mask) != p->val)
2305	p++;
2306	ver = p->ver;
2307
2308	if (ver != RTL_GIGA_MAC_NONE && !gmii) {
2309	if (ver == RTL_GIGA_MAC_VER_42)
2310	ver = RTL_GIGA_MAC_VER_43;
2311	else if (ver == RTL_GIGA_MAC_VER_46)
2312	ver = RTL_GIGA_MAC_VER_48;
2313	}
2314
2315	return ver;
2316	}
2317
2318	static void rtl_release_firmware(struct rtl8169_private *tp)
2319	{
2320	if (tp->rtl_fw) {
2321	rtl_fw_release_firmware(rtl_fw: tp->rtl_fw);
2322	kfree(objp: tp->rtl_fw);
2323	tp->rtl_fw = NULL;
2324	}
2325	}
2326
2327	void r8169_apply_firmware(struct rtl8169_private *tp)
2328	{
2329	int val;
2330
2331	/* TODO: release firmware if rtl_fw_write_firmware signals failure. */
2332	if (tp->rtl_fw) {
2333	rtl_fw_write_firmware(tp, rtl_fw: tp->rtl_fw);
2334	/* At least one firmware doesn't reset tp->ocp_base. */
2335	tp->ocp_base = OCP_STD_PHY_BASE;
2336
2337	/* PHY soft reset may still be in progress */
2338	phy_read_poll_timeout(tp->phydev, MII_BMCR, val,
2339	!(val & BMCR_RESET),
2340	50000, 600000, true);
2341	}
2342	}
2343
2344	static void rtl8168_config_eee_mac(struct rtl8169_private *tp)
2345	{
2346	/* Adjust EEE LED frequency */
2347	if (tp->mac_version != RTL_GIGA_MAC_VER_38)
2348	RTL_W8(tp, EEE_LED, RTL_R8(tp, EEE_LED) & ~0x07);
2349
2350	rtl_eri_set_bits(tp, addr: 0x1b0, p: 0x0003);
2351	}
2352
2353	static void rtl8125a_config_eee_mac(struct rtl8169_private *tp)
2354	{
2355	r8168_mac_ocp_modify(tp, reg: 0xe040, mask: 0, BIT(1) | BIT(0));
2356	r8168_mac_ocp_modify(tp, reg: 0xeb62, mask: 0, BIT(2) | BIT(1));
2357	}
2358
2359	static void rtl8125b_config_eee_mac(struct rtl8169_private *tp)
2360	{
2361	r8168_mac_ocp_modify(tp, reg: 0xe040, mask: 0, BIT(1) | BIT(0));
2362	}
2363
2364	static void rtl_rar_exgmac_set(struct rtl8169_private *tp, const u8 *addr)
2365	{
2366	rtl_eri_write(tp, addr: 0xe0, ERIAR_MASK_1111, val: get_unaligned_le32(p: addr));
2367	rtl_eri_write(tp, addr: 0xe4, ERIAR_MASK_1111, val: get_unaligned_le16(p: addr + 4));
2368	rtl_eri_write(tp, addr: 0xf0, ERIAR_MASK_1111, val: get_unaligned_le16(p: addr) << 16);
2369	rtl_eri_write(tp, addr: 0xf4, ERIAR_MASK_1111, val: get_unaligned_le32(p: addr + 2));
2370	}
2371
2372	u16 rtl8168h_2_get_adc_bias_ioffset(struct rtl8169_private *tp)
2373	{
2374	u16 data1, data2, ioffset;
2375
2376	r8168_mac_ocp_write(tp, reg: 0xdd02, data: 0x807d);
2377	data1 = r8168_mac_ocp_read(tp, reg: 0xdd02);
2378	data2 = r8168_mac_ocp_read(tp, reg: 0xdd00);
2379
2380	ioffset = (data2 >> 1) & 0x7ff8;
2381	ioffset |= data2 & 0x0007;
2382	if (data1 & BIT(7))
2383	ioffset |= BIT(15);
2384
2385	return ioffset;
2386	}
2387
2388	static void rtl_schedule_task(struct rtl8169_private *tp, enum rtl_flag flag)
2389	{
2390	if (!test_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags))
2391	return;
2392
2393	set_bit(nr: flag, addr: tp->wk.flags);
2394	schedule_work(work: &tp->wk.work);
2395	}
2396
2397	static void rtl8169_init_phy(struct rtl8169_private *tp)
2398	{
2399	r8169_hw_phy_config(tp, phydev: tp->phydev, ver: tp->mac_version);
2400
2401	if (tp->mac_version <= RTL_GIGA_MAC_VER_06) {
2402	pci_write_config_byte(dev: tp->pci_dev, PCI_LATENCY_TIMER, val: 0x40);
2403	pci_write_config_byte(dev: tp->pci_dev, PCI_CACHE_LINE_SIZE, val: 0x08);
2404	/* set undocumented MAC Reg C+CR Offset 0x82h */
2405	RTL_W8(tp, 0x82, 0x01);
2406	}
2407
2408	if (tp->mac_version == RTL_GIGA_MAC_VER_05 &&
2409	tp->pci_dev->subsystem_vendor == PCI_VENDOR_ID_GIGABYTE &&
2410	tp->pci_dev->subsystem_device == 0xe000)
2411	phy_write_paged(phydev: tp->phydev, page: 0x0001, regnum: 0x10, val: 0xf01b);
2412
2413	/* We may have called phy_speed_down before */
2414	phy_speed_up(phydev: tp->phydev);
2415
2416	genphy_soft_reset(phydev: tp->phydev);
2417	}
2418
2419	static void rtl_rar_set(struct rtl8169_private *tp, const u8 *addr)
2420	{
2421	rtl_unlock_config_regs(tp);
2422
2423	RTL_W32(tp, MAC4, get_unaligned_le16(addr + 4));
2424	rtl_pci_commit(tp);
2425
2426	RTL_W32(tp, MAC0, get_unaligned_le32(addr));
2427	rtl_pci_commit(tp);
2428
2429	if (tp->mac_version == RTL_GIGA_MAC_VER_34)
2430	rtl_rar_exgmac_set(tp, addr);
2431
2432	rtl_lock_config_regs(tp);
2433	}
2434
2435	static int rtl_set_mac_address(struct net_device *dev, void *p)
2436	{
2437	struct rtl8169_private *tp = netdev_priv(dev);
2438	int ret;
2439
2440	ret = eth_mac_addr(dev, p);
2441	if (ret)
2442	return ret;
2443
2444	rtl_rar_set(tp, addr: dev->dev_addr);
2445
2446	return 0;
2447	}
2448
2449	static void rtl_init_rxcfg(struct rtl8169_private *tp)
2450	{
2451	switch (tp->mac_version) {
2452	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
2453	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
2454	RTL_W32(tp, RxConfig, RX_FIFO_THRESH | RX_DMA_BURST);
2455	break;
2456	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
2457	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2458	case RTL_GIGA_MAC_VER_38:
2459	RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST);
2460	break;
2461	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
2462	RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF);
2463	break;
2464	case RTL_GIGA_MAC_VER_61:
2465	RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST);
2466	break;
2467	case RTL_GIGA_MAC_VER_63:
2468	case RTL_GIGA_MAC_VER_65:
2469	RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST |
2470	RX_PAUSE_SLOT_ON);
2471	break;
2472	default:
2473	RTL_W32(tp, RxConfig, RX128_INT_EN | RX_DMA_BURST);
2474	break;
2475	}
2476	}
2477
2478	static void rtl8169_init_ring_indexes(struct rtl8169_private *tp)
2479	{
2480	tp->dirty_tx = tp->cur_tx = tp->cur_rx = 0;
2481	}
2482
2483	static void r8168c_hw_jumbo_enable(struct rtl8169_private *tp)
2484	{
2485	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2486	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | Jumbo_En1);
2487	}
2488
2489	static void r8168c_hw_jumbo_disable(struct rtl8169_private *tp)
2490	{
2491	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2492	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~Jumbo_En1);
2493	}
2494
2495	static void r8168dp_hw_jumbo_enable(struct rtl8169_private *tp)
2496	{
2497	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2498	}
2499
2500	static void r8168dp_hw_jumbo_disable(struct rtl8169_private *tp)
2501	{
2502	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2503	}
2504
2505	static void r8168e_hw_jumbo_enable(struct rtl8169_private *tp)
2506	{
2507	RTL_W8(tp, MaxTxPacketSize, 0x24);
2508	RTL_W8(tp, Config3, RTL_R8(tp, Config3) | Jumbo_En0);
2509	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | 0x01);
2510	}
2511
2512	static void r8168e_hw_jumbo_disable(struct rtl8169_private *tp)
2513	{
2514	RTL_W8(tp, MaxTxPacketSize, 0x3f);
2515	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Jumbo_En0);
2516	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~0x01);
2517	}
2518
2519	static void r8168b_1_hw_jumbo_enable(struct rtl8169_private *tp)
2520	{
2521	RTL_W8(tp, Config4, RTL_R8(tp, Config4) | (1 << 0));
2522	}
2523
2524	static void r8168b_1_hw_jumbo_disable(struct rtl8169_private *tp)
2525	{
2526	RTL_W8(tp, Config4, RTL_R8(tp, Config4) & ~(1 << 0));
2527	}
2528
2529	static void rtl_jumbo_config(struct rtl8169_private *tp)
2530	{
2531	bool jumbo = tp->dev->mtu > ETH_DATA_LEN;
2532	int readrq = 4096;
2533
2534	rtl_unlock_config_regs(tp);
2535	switch (tp->mac_version) {
2536	case RTL_GIGA_MAC_VER_17:
2537	if (jumbo) {
2538	readrq = 512;
2539	r8168b_1_hw_jumbo_enable(tp);
2540	} else {
2541	r8168b_1_hw_jumbo_disable(tp);
2542	}
2543	break;
2544	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_26:
2545	if (jumbo) {
2546	readrq = 512;
2547	r8168c_hw_jumbo_enable(tp);
2548	} else {
2549	r8168c_hw_jumbo_disable(tp);
2550	}
2551	break;
2552	case RTL_GIGA_MAC_VER_28:
2553	if (jumbo)
2554	r8168dp_hw_jumbo_enable(tp);
2555	else
2556	r8168dp_hw_jumbo_disable(tp);
2557	break;
2558	case RTL_GIGA_MAC_VER_31 ... RTL_GIGA_MAC_VER_33:
2559	if (jumbo)
2560	r8168e_hw_jumbo_enable(tp);
2561	else
2562	r8168e_hw_jumbo_disable(tp);
2563	break;
2564	default:
2565	break;
2566	}
2567	rtl_lock_config_regs(tp);
2568
2569	if (pci_is_pcie(dev: tp->pci_dev) && tp->supports_gmii)
2570	pcie_set_readrq(dev: tp->pci_dev, rq: readrq);
2571
2572	/* Chip doesn't support pause in jumbo mode */
2573	if (jumbo) {
2574	linkmode_clear_bit(nr: ETHTOOL_LINK_MODE_Pause_BIT,
2575	addr: tp->phydev->advertising);
2576	linkmode_clear_bit(nr: ETHTOOL_LINK_MODE_Asym_Pause_BIT,
2577	addr: tp->phydev->advertising);
2578	phy_start_aneg(phydev: tp->phydev);
2579	}
2580	}
2581
2582	DECLARE_RTL_COND(rtl_chipcmd_cond)
2583	{
2584	return RTL_R8(tp, ChipCmd) & CmdReset;
2585	}
2586
2587	static void rtl_hw_reset(struct rtl8169_private *tp)
2588	{
2589	RTL_W8(tp, ChipCmd, CmdReset);
2590
2591	rtl_loop_wait_low(tp, c: &rtl_chipcmd_cond, d: 100, n: 100);
2592	}
2593
2594	static void rtl_request_firmware(struct rtl8169_private *tp)
2595	{
2596	struct rtl_fw *rtl_fw;
2597
2598	/* firmware loaded already or no firmware available */
2599	if (tp->rtl_fw || !tp->fw_name)
2600	return;
2601
2602	rtl_fw = kzalloc(size: sizeof(*rtl_fw), GFP_KERNEL);
2603	if (!rtl_fw)
2604	return;
2605
2606	rtl_fw->phy_write = rtl_writephy;
2607	rtl_fw->phy_read = rtl_readphy;
2608	rtl_fw->mac_mcu_write = mac_mcu_write;
2609	rtl_fw->mac_mcu_read = mac_mcu_read;
2610	rtl_fw->fw_name = tp->fw_name;
2611	rtl_fw->dev = tp_to_dev(tp);
2612
2613	if (rtl_fw_request_firmware(rtl_fw))
2614	kfree(objp: rtl_fw);
2615	else
2616	tp->rtl_fw = rtl_fw;
2617	}
2618
2619	static void rtl_rx_close(struct rtl8169_private *tp)
2620	{
2621	RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) & ~RX_CONFIG_ACCEPT_MASK);
2622	}
2623
2624	DECLARE_RTL_COND(rtl_npq_cond)
2625	{
2626	return RTL_R8(tp, TxPoll) & NPQ;
2627	}
2628
2629	DECLARE_RTL_COND(rtl_txcfg_empty_cond)
2630	{
2631	return RTL_R32(tp, TxConfig) & TXCFG_EMPTY;
2632	}
2633
2634	DECLARE_RTL_COND(rtl_rxtx_empty_cond)
2635	{
2636	return (RTL_R8(tp, MCU) & RXTX_EMPTY) == RXTX_EMPTY;
2637	}
2638
2639	DECLARE_RTL_COND(rtl_rxtx_empty_cond_2)
2640	{
2641	/* IntrMitigate has new functionality on RTL8125 */
2642	return (RTL_R16(tp, IntrMitigate) & 0x0103) == 0x0103;
2643	}
2644
2645	static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp)
2646	{
2647	switch (tp->mac_version) {
2648	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
2649	rtl_loop_wait_high(tp, c: &rtl_txcfg_empty_cond, d: 100, n: 42);
2650	rtl_loop_wait_high(tp, c: &rtl_rxtx_empty_cond, d: 100, n: 42);
2651	break;
2652	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_61:
2653	rtl_loop_wait_high(tp, c: &rtl_rxtx_empty_cond, d: 100, n: 42);
2654	break;
2655	case RTL_GIGA_MAC_VER_63 ... RTL_GIGA_MAC_VER_65:
2656	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
2657	rtl_loop_wait_high(tp, c: &rtl_rxtx_empty_cond, d: 100, n: 42);
2658	rtl_loop_wait_high(tp, c: &rtl_rxtx_empty_cond_2, d: 100, n: 42);
2659	break;
2660	default:
2661	break;
2662	}
2663	}
2664
2665	static void rtl_disable_rxdvgate(struct rtl8169_private *tp)
2666	{
2667	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
2668	}
2669
2670	static void rtl_enable_rxdvgate(struct rtl8169_private *tp)
2671	{
2672	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | RXDV_GATED_EN);
2673	fsleep(usecs: 2000);
2674	rtl_wait_txrx_fifo_empty(tp);
2675	}
2676
2677	static void rtl_wol_enable_rx(struct rtl8169_private *tp)
2678	{
2679	if (tp->mac_version >= RTL_GIGA_MAC_VER_25)
2680	RTL_W32(tp, RxConfig, RTL_R32(tp, RxConfig) |
2681	AcceptBroadcast | AcceptMulticast | AcceptMyPhys);
2682
2683	if (tp->mac_version >= RTL_GIGA_MAC_VER_40)
2684	rtl_disable_rxdvgate(tp);
2685	}
2686
2687	static void rtl_prepare_power_down(struct rtl8169_private *tp)
2688	{
2689	if (tp->dash_enabled)
2690	return;
2691
2692	if (tp->mac_version == RTL_GIGA_MAC_VER_32 ||
2693	tp->mac_version == RTL_GIGA_MAC_VER_33)
2694	rtl_ephy_write(tp, reg_addr: 0x19, value: 0xff64);
2695
2696	if (device_may_wakeup(dev: tp_to_dev(tp))) {
2697	phy_speed_down(phydev: tp->phydev, sync: false);
2698	rtl_wol_enable_rx(tp);
2699	}
2700	}
2701
2702	static void rtl_set_tx_config_registers(struct rtl8169_private *tp)
2703	{
2704	u32 val = TX_DMA_BURST << TxDMAShift |
2705	InterFrameGap << TxInterFrameGapShift;
2706
2707	if (rtl_is_8168evl_up(tp))
2708	val |= TXCFG_AUTO_FIFO;
2709
2710	RTL_W32(tp, TxConfig, val);
2711	}
2712
2713	static void rtl_set_rx_max_size(struct rtl8169_private *tp)
2714	{
2715	/* Low hurts. Let's disable the filtering. */
2716	RTL_W16(tp, RxMaxSize, R8169_RX_BUF_SIZE + 1);
2717	}
2718
2719	static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp)
2720	{
2721	/*
2722	* Magic spell: some iop3xx ARM board needs the TxDescAddrHigh
2723	* register to be written before TxDescAddrLow to work.
2724	* Switching from MMIO to I/O access fixes the issue as well.
2725	*/
2726	RTL_W32(tp, TxDescStartAddrHigh, ((u64) tp->TxPhyAddr) >> 32);
2727	RTL_W32(tp, TxDescStartAddrLow, ((u64) tp->TxPhyAddr) & DMA_BIT_MASK(32));
2728	RTL_W32(tp, RxDescAddrHigh, ((u64) tp->RxPhyAddr) >> 32);
2729	RTL_W32(tp, RxDescAddrLow, ((u64) tp->RxPhyAddr) & DMA_BIT_MASK(32));
2730	}
2731
2732	static void rtl8169_set_magic_reg(struct rtl8169_private *tp)
2733	{
2734	u32 val;
2735
2736	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
2737	val = 0x000fff00;
2738	else if (tp->mac_version == RTL_GIGA_MAC_VER_06)
2739	val = 0x00ffff00;
2740	else
2741	return;
2742
2743	if (RTL_R8(tp, Config2) & PCI_Clock_66MHz)
2744	val |= 0xff;
2745
2746	RTL_W32(tp, 0x7c, val);
2747	}
2748
2749	static void rtl_set_rx_mode(struct net_device *dev)
2750	{
2751	u32 rx_mode = AcceptBroadcast | AcceptMyPhys | AcceptMulticast;
2752	/* Multicast hash filter */
2753	u32 mc_filter[2] = { 0xffffffff, 0xffffffff };
2754	struct rtl8169_private *tp = netdev_priv(dev);
2755	u32 tmp;
2756
2757	if (dev->flags & IFF_PROMISC) {
2758	rx_mode |= AcceptAllPhys;
2759	} else if (!(dev->flags & IFF_MULTICAST)) {
2760	rx_mode &= ~AcceptMulticast;
2761	} else if (dev->flags & IFF_ALLMULTI ||
2762	tp->mac_version == RTL_GIGA_MAC_VER_35) {
2763	/* accept all multicasts */
2764	} else if (netdev_mc_empty(dev)) {
2765	rx_mode &= ~AcceptMulticast;
2766	} else {
2767	struct netdev_hw_addr *ha;
2768
2769	mc_filter[1] = mc_filter[0] = 0;
2770	netdev_for_each_mc_addr(ha, dev) {
2771	u32 bit_nr = eth_hw_addr_crc(ha) >> 26;
2772	mc_filter[bit_nr >> 5] |= BIT(bit_nr & 31);
2773	}
2774
2775	if (tp->mac_version > RTL_GIGA_MAC_VER_06) {
2776	tmp = mc_filter[0];
2777	mc_filter[0] = swab32(mc_filter[1]);
2778	mc_filter[1] = swab32(tmp);
2779	}
2780	}
2781
2782	RTL_W32(tp, MAR0 + 4, mc_filter[1]);
2783	RTL_W32(tp, MAR0 + 0, mc_filter[0]);
2784
2785	tmp = RTL_R32(tp, RxConfig);
2786	RTL_W32(tp, RxConfig, (tmp & ~RX_CONFIG_ACCEPT_OK_MASK) | rx_mode);
2787	}
2788
2789	DECLARE_RTL_COND(rtl_csiar_cond)
2790	{
2791	return RTL_R32(tp, CSIAR) & CSIAR_FLAG;
2792	}
2793
2794	static void rtl_csi_write(struct rtl8169_private *tp, int addr, int value)
2795	{
2796	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2797
2798	RTL_W32(tp, CSIDR, value);
2799	RTL_W32(tp, CSIAR, CSIAR_WRITE_CMD | (addr & CSIAR_ADDR_MASK) |
2800	CSIAR_BYTE_ENABLE | func << 16);
2801
2802	rtl_loop_wait_low(tp, c: &rtl_csiar_cond, d: 10, n: 100);
2803	}
2804
2805	static u32 rtl_csi_read(struct rtl8169_private *tp, int addr)
2806	{
2807	u32 func = PCI_FUNC(tp->pci_dev->devfn);
2808
2809	RTL_W32(tp, CSIAR, (addr & CSIAR_ADDR_MASK) | func << 16 |
2810	CSIAR_BYTE_ENABLE);
2811
2812	return rtl_loop_wait_high(tp, c: &rtl_csiar_cond, d: 10, n: 100) ?
2813	RTL_R32(tp, CSIDR) : ~0;
2814	}
2815
2816	static void rtl_set_aspm_entry_latency(struct rtl8169_private *tp, u8 val)
2817	{
2818	struct pci_dev *pdev = tp->pci_dev;
2819	u32 csi;
2820
2821	/* According to Realtek the value at config space address 0x070f
2822	* controls the L0s/L1 entrance latency. We try standard ECAM access
2823	* first and if it fails fall back to CSI.
2824	* bit 0..2: L0: 0 = 1us, 1 = 2us .. 6 = 7us, 7 = 7us (no typo)
2825	* bit 3..5: L1: 0 = 1us, 1 = 2us .. 6 = 64us, 7 = 64us
2826	*/
2827	if (pdev->cfg_size > 0x070f &&
2828	pci_write_config_byte(dev: pdev, where: 0x070f, val) == PCIBIOS_SUCCESSFUL)
2829	return;
2830
2831	netdev_notice_once(tp->dev,
2832	"No native access to PCI extended config space, falling back to CSI\n");
2833	csi = rtl_csi_read(tp, addr: 0x070c) & 0x00ffffff;
2834	rtl_csi_write(tp, addr: 0x070c, value: csi | val << 24);
2835	}
2836
2837	static void rtl_set_def_aspm_entry_latency(struct rtl8169_private *tp)
2838	{
2839	/* L0 7us, L1 16us */
2840	rtl_set_aspm_entry_latency(tp, val: 0x27);
2841	}
2842
2843	struct ephy_info {
2844	unsigned int offset;
2845	u16 mask;
2846	u16 bits;
2847	};
2848
2849	static void __rtl_ephy_init(struct rtl8169_private *tp,
2850	const struct ephy_info *e, int len)
2851	{
2852	u16 w;
2853
2854	while (len-- > 0) {
2855	w = (rtl_ephy_read(tp, reg_addr: e->offset) & ~e->mask) | e->bits;
2856	rtl_ephy_write(tp, reg_addr: e->offset, value: w);
2857	e++;
2858	}
2859	}
2860
2861	#define rtl_ephy_init(tp, a) __rtl_ephy_init(tp, a, ARRAY_SIZE(a))
2862
2863	static void rtl_disable_clock_request(struct rtl8169_private *tp)
2864	{
2865	pcie_capability_clear_word(dev: tp->pci_dev, PCI_EXP_LNKCTL,
2866	PCI_EXP_LNKCTL_CLKREQ_EN);
2867	}
2868
2869	static void rtl_enable_clock_request(struct rtl8169_private *tp)
2870	{
2871	pcie_capability_set_word(dev: tp->pci_dev, PCI_EXP_LNKCTL,
2872	PCI_EXP_LNKCTL_CLKREQ_EN);
2873	}
2874
2875	static void rtl_pcie_state_l2l3_disable(struct rtl8169_private *tp)
2876	{
2877	/* work around an issue when PCI reset occurs during L2/L3 state */
2878	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Rdy_to_L23);
2879	}
2880
2881	static void rtl_enable_exit_l1(struct rtl8169_private *tp)
2882	{
2883	/* Bits control which events trigger ASPM L1 exit:
2884	* Bit 12: rxdv
2885	* Bit 11: ltr_msg
2886	* Bit 10: txdma_poll
2887	* Bit 9: xadm
2888	* Bit 8: pktavi
2889	* Bit 7: txpla
2890	*/
2891	switch (tp->mac_version) {
2892	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_36:
2893	rtl_eri_set_bits(tp, addr: 0xd4, p: 0x1f00);
2894	break;
2895	case RTL_GIGA_MAC_VER_37 ... RTL_GIGA_MAC_VER_38:
2896	rtl_eri_set_bits(tp, addr: 0xd4, p: 0x0c00);
2897	break;
2898	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_65:
2899	r8168_mac_ocp_modify(tp, reg: 0xc0ac, mask: 0, set: 0x1f80);
2900	break;
2901	default:
2902	break;
2903	}
2904	}
2905
2906	static void rtl_disable_exit_l1(struct rtl8169_private *tp)
2907	{
2908	switch (tp->mac_version) {
2909	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
2910	rtl_eri_clear_bits(tp, addr: 0xd4, m: 0x1f00);
2911	break;
2912	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_65:
2913	r8168_mac_ocp_modify(tp, reg: 0xc0ac, mask: 0x1f80, set: 0);
2914	break;
2915	default:
2916	break;
2917	}
2918	}
2919
2920	static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
2921	{
2922	u8 val8;
2923
2924	if (tp->mac_version < RTL_GIGA_MAC_VER_32)
2925	return;
2926
2927	/* Don't enable ASPM in the chip if OS can't control ASPM */
2928	if (enable && tp->aspm_manageable) {
2929	/* On these chip versions ASPM can even harm
2930	* bus communication of other PCI devices.
2931	*/
2932	if (tp->mac_version == RTL_GIGA_MAC_VER_42 ||
2933	tp->mac_version == RTL_GIGA_MAC_VER_43)
2934	return;
2935
2936	rtl_mod_config5(tp, clear: 0, set: ASPM_en);
2937	switch (tp->mac_version) {
2938	case RTL_GIGA_MAC_VER_65:
2939	val8 = RTL_R8(tp, INT_CFG0_8125) | INT_CFG0_CLKREQEN;
2940	RTL_W8(tp, INT_CFG0_8125, val8);
2941	break;
2942	default:
2943	rtl_mod_config2(tp, clear: 0, set: ClkReqEn);
2944	break;
2945	}
2946
2947	switch (tp->mac_version) {
2948	case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
2949	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_65:
2950	/* reset ephy tx/rx disable timer */
2951	r8168_mac_ocp_modify(tp, reg: 0xe094, mask: 0xff00, set: 0);
2952	/* chip can trigger L1.2 */
2953	r8168_mac_ocp_modify(tp, reg: 0xe092, mask: 0x00ff, BIT(2));
2954	break;
2955	default:
2956	break;
2957	}
2958	} else {
2959	switch (tp->mac_version) {
2960	case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
2961	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_65:
2962	r8168_mac_ocp_modify(tp, reg: 0xe092, mask: 0x00ff, set: 0);
2963	break;
2964	default:
2965	break;
2966	}
2967
2968	switch (tp->mac_version) {
2969	case RTL_GIGA_MAC_VER_65:
2970	val8 = RTL_R8(tp, INT_CFG0_8125) & ~INT_CFG0_CLKREQEN;
2971	RTL_W8(tp, INT_CFG0_8125, val8);
2972	break;
2973	default:
2974	rtl_mod_config2(tp, clear: ClkReqEn, set: 0);
2975	break;
2976	}
2977	rtl_mod_config5(tp, clear: ASPM_en, set: 0);
2978	}
2979	}
2980
2981	static void rtl_set_fifo_size(struct rtl8169_private *tp, u16 rx_stat,
2982	u16 tx_stat, u16 rx_dyn, u16 tx_dyn)
2983	{
2984	/* Usage of dynamic vs. static FIFO is controlled by bit
2985	* TXCFG_AUTO_FIFO. Exact meaning of FIFO values isn't known.
2986	*/
2987	rtl_eri_write(tp, addr: 0xc8, ERIAR_MASK_1111, val: (rx_stat << 16) | rx_dyn);
2988	rtl_eri_write(tp, addr: 0xe8, ERIAR_MASK_1111, val: (tx_stat << 16) | tx_dyn);
2989	}
2990
2991	static void rtl8168g_set_pause_thresholds(struct rtl8169_private *tp,
2992	u8 low, u8 high)
2993	{
2994	/* FIFO thresholds for pause flow control */
2995	rtl_eri_write(tp, addr: 0xcc, ERIAR_MASK_0001, val: low);
2996	rtl_eri_write(tp, addr: 0xd0, ERIAR_MASK_0001, val: high);
2997	}
2998
2999	static void rtl_hw_start_8168b(struct rtl8169_private *tp)
3000	{
3001	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3002	}
3003
3004	static void __rtl_hw_start_8168cp(struct rtl8169_private *tp)
3005	{
3006	RTL_W8(tp, Config1, RTL_R8(tp, Config1) | Speed_down);
3007
3008	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3009
3010	rtl_disable_clock_request(tp);
3011	}
3012
3013	static void rtl_hw_start_8168cp_1(struct rtl8169_private *tp)
3014	{
3015	static const struct ephy_info e_info_8168cp[] = {
3016	{ 0x01, 0, 0x0001 },
3017	{ 0x02, 0x0800, 0x1000 },
3018	{ 0x03, 0, 0x0042 },
3019	{ 0x06, 0x0080, 0x0000 },
3020	{ 0x07, 0, 0x2000 }
3021	};
3022
3023	rtl_set_def_aspm_entry_latency(tp);
3024
3025	rtl_ephy_init(tp, e_info_8168cp);
3026
3027	__rtl_hw_start_8168cp(tp);
3028	}
3029
3030	static void rtl_hw_start_8168cp_2(struct rtl8169_private *tp)
3031	{
3032	rtl_set_def_aspm_entry_latency(tp);
3033
3034	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3035	}
3036
3037	static void rtl_hw_start_8168cp_3(struct rtl8169_private *tp)
3038	{
3039	rtl_set_def_aspm_entry_latency(tp);
3040
3041	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3042
3043	/* Magic. */
3044	RTL_W8(tp, DBG_REG, 0x20);
3045	}
3046
3047	static void rtl_hw_start_8168c_1(struct rtl8169_private *tp)
3048	{
3049	static const struct ephy_info e_info_8168c_1[] = {
3050	{ 0x02, 0x0800, 0x1000 },
3051	{ 0x03, 0, 0x0002 },
3052	{ 0x06, 0x0080, 0x0000 }
3053	};
3054
3055	rtl_set_def_aspm_entry_latency(tp);
3056
3057	RTL_W8(tp, DBG_REG, 0x06 | FIX_NAK_1 | FIX_NAK_2);
3058
3059	rtl_ephy_init(tp, e_info_8168c_1);
3060
3061	__rtl_hw_start_8168cp(tp);
3062	}
3063
3064	static void rtl_hw_start_8168c_2(struct rtl8169_private *tp)
3065	{
3066	static const struct ephy_info e_info_8168c_2[] = {
3067	{ 0x01, 0, 0x0001 },
3068	{ 0x03, 0x0400, 0x0020 }
3069	};
3070
3071	rtl_set_def_aspm_entry_latency(tp);
3072
3073	rtl_ephy_init(tp, e_info_8168c_2);
3074
3075	__rtl_hw_start_8168cp(tp);
3076	}
3077
3078	static void rtl_hw_start_8168c_4(struct rtl8169_private *tp)
3079	{
3080	rtl_set_def_aspm_entry_latency(tp);
3081
3082	__rtl_hw_start_8168cp(tp);
3083	}
3084
3085	static void rtl_hw_start_8168d(struct rtl8169_private *tp)
3086	{
3087	rtl_set_def_aspm_entry_latency(tp);
3088
3089	rtl_disable_clock_request(tp);
3090	}
3091
3092	static void rtl_hw_start_8168d_4(struct rtl8169_private *tp)
3093	{
3094	static const struct ephy_info e_info_8168d_4[] = {
3095	{ 0x0b, 0x0000, 0x0048 },
3096	{ 0x19, 0x0020, 0x0050 },
3097	{ 0x0c, 0x0100, 0x0020 },
3098	{ 0x10, 0x0004, 0x0000 },
3099	};
3100
3101	rtl_set_def_aspm_entry_latency(tp);
3102
3103	rtl_ephy_init(tp, e_info_8168d_4);
3104
3105	rtl_enable_clock_request(tp);
3106	}
3107
3108	static void rtl_hw_start_8168e_1(struct rtl8169_private *tp)
3109	{
3110	static const struct ephy_info e_info_8168e_1[] = {
3111	{ 0x00, 0x0200, 0x0100 },
3112	{ 0x00, 0x0000, 0x0004 },
3113	{ 0x06, 0x0002, 0x0001 },
3114	{ 0x06, 0x0000, 0x0030 },
3115	{ 0x07, 0x0000, 0x2000 },
3116	{ 0x00, 0x0000, 0x0020 },
3117	{ 0x03, 0x5800, 0x2000 },
3118	{ 0x03, 0x0000, 0x0001 },
3119	{ 0x01, 0x0800, 0x1000 },
3120	{ 0x07, 0x0000, 0x4000 },
3121	{ 0x1e, 0x0000, 0x2000 },
3122	{ 0x19, 0xffff, 0xfe6c },
3123	{ 0x0a, 0x0000, 0x0040 }
3124	};
3125
3126	rtl_set_def_aspm_entry_latency(tp);
3127
3128	rtl_ephy_init(tp, e_info_8168e_1);
3129
3130	rtl_disable_clock_request(tp);
3131
3132	/* Reset tx FIFO pointer */
3133	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | TXPLA_RST);
3134	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~TXPLA_RST);
3135
3136	rtl_mod_config5(tp, clear: Spi_en, set: 0);
3137	}
3138
3139	static void rtl_hw_start_8168e_2(struct rtl8169_private *tp)
3140	{
3141	static const struct ephy_info e_info_8168e_2[] = {
3142	{ 0x09, 0x0000, 0x0080 },
3143	{ 0x19, 0x0000, 0x0224 },
3144	{ 0x00, 0x0000, 0x0004 },
3145	{ 0x0c, 0x3df0, 0x0200 },
3146	};
3147
3148	rtl_set_def_aspm_entry_latency(tp);
3149
3150	rtl_ephy_init(tp, e_info_8168e_2);
3151
3152	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3153	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_1111, val: 0x0000);
3154	rtl_set_fifo_size(tp, rx_stat: 0x10, tx_stat: 0x10, rx_dyn: 0x02, tx_dyn: 0x06);
3155	rtl_eri_set_bits(tp, addr: 0x1d0, BIT(1));
3156	rtl_reset_packet_filter(tp);
3157	rtl_eri_set_bits(tp, addr: 0x1b0, BIT(4));
3158	rtl_eri_write(tp, addr: 0xcc, ERIAR_MASK_1111, val: 0x00000050);
3159	rtl_eri_write(tp, addr: 0xd0, ERIAR_MASK_1111, val: 0x07ff0060);
3160
3161	rtl_disable_clock_request(tp);
3162
3163	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
3164
3165	rtl8168_config_eee_mac(tp);
3166
3167	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
3168	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
3169	rtl_mod_config5(tp, clear: Spi_en, set: 0);
3170	}
3171
3172	static void rtl_hw_start_8168f(struct rtl8169_private *tp)
3173	{
3174	rtl_set_def_aspm_entry_latency(tp);
3175
3176	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3177	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_1111, val: 0x0000);
3178	rtl_set_fifo_size(tp, rx_stat: 0x10, tx_stat: 0x10, rx_dyn: 0x02, tx_dyn: 0x06);
3179	rtl_reset_packet_filter(tp);
3180	rtl_eri_set_bits(tp, addr: 0x1b0, BIT(4));
3181	rtl_eri_set_bits(tp, addr: 0x1d0, BIT(4) | BIT(1));
3182	rtl_eri_write(tp, addr: 0xcc, ERIAR_MASK_1111, val: 0x00000050);
3183	rtl_eri_write(tp, addr: 0xd0, ERIAR_MASK_1111, val: 0x00000060);
3184
3185	rtl_disable_clock_request(tp);
3186
3187	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
3188	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
3189	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | PWM_EN);
3190	rtl_mod_config5(tp, clear: Spi_en, set: 0);
3191
3192	rtl8168_config_eee_mac(tp);
3193	}
3194
3195	static void rtl_hw_start_8168f_1(struct rtl8169_private *tp)
3196	{
3197	static const struct ephy_info e_info_8168f_1[] = {
3198	{ 0x06, 0x00c0, 0x0020 },
3199	{ 0x08, 0x0001, 0x0002 },
3200	{ 0x09, 0x0000, 0x0080 },
3201	{ 0x19, 0x0000, 0x0224 },
3202	{ 0x00, 0x0000, 0x0008 },
3203	{ 0x0c, 0x3df0, 0x0200 },
3204	};
3205
3206	rtl_hw_start_8168f(tp);
3207
3208	rtl_ephy_init(tp, e_info_8168f_1);
3209	}
3210
3211	static void rtl_hw_start_8411(struct rtl8169_private *tp)
3212	{
3213	static const struct ephy_info e_info_8168f_1[] = {
3214	{ 0x06, 0x00c0, 0x0020 },
3215	{ 0x0f, 0xffff, 0x5200 },
3216	{ 0x19, 0x0000, 0x0224 },
3217	{ 0x00, 0x0000, 0x0008 },
3218	{ 0x0c, 0x3df0, 0x0200 },
3219	};
3220
3221	rtl_hw_start_8168f(tp);
3222	rtl_pcie_state_l2l3_disable(tp);
3223
3224	rtl_ephy_init(tp, e_info_8168f_1);
3225	}
3226
3227	static void rtl_hw_start_8168g(struct rtl8169_private *tp)
3228	{
3229	rtl_set_fifo_size(tp, rx_stat: 0x08, tx_stat: 0x10, rx_dyn: 0x02, tx_dyn: 0x06);
3230	rtl8168g_set_pause_thresholds(tp, low: 0x38, high: 0x48);
3231
3232	rtl_set_def_aspm_entry_latency(tp);
3233
3234	rtl_reset_packet_filter(tp);
3235	rtl_eri_write(tp, addr: 0x2f8, ERIAR_MASK_0011, val: 0x1d8f);
3236
3237	rtl_disable_rxdvgate(tp);
3238
3239	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3240	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_0011, val: 0x0000);
3241
3242	rtl8168_config_eee_mac(tp);
3243
3244	rtl_w0w1_eri(tp, addr: 0x2fc, p: 0x01, m: 0x06);
3245	rtl_eri_clear_bits(tp, addr: 0x1b0, BIT(12));
3246
3247	rtl_pcie_state_l2l3_disable(tp);
3248	}
3249
3250	static void rtl_hw_start_8168g_1(struct rtl8169_private *tp)
3251	{
3252	static const struct ephy_info e_info_8168g_1[] = {
3253	{ 0x00, 0x0008, 0x0000 },
3254	{ 0x0c, 0x3ff0, 0x0820 },
3255	{ 0x1e, 0x0000, 0x0001 },
3256	{ 0x19, 0x8000, 0x0000 }
3257	};
3258
3259	rtl_hw_start_8168g(tp);
3260	rtl_ephy_init(tp, e_info_8168g_1);
3261	}
3262
3263	static void rtl_hw_start_8168g_2(struct rtl8169_private *tp)
3264	{
3265	static const struct ephy_info e_info_8168g_2[] = {
3266	{ 0x00, 0x0008, 0x0000 },
3267	{ 0x0c, 0x3ff0, 0x0820 },
3268	{ 0x19, 0xffff, 0x7c00 },
3269	{ 0x1e, 0xffff, 0x20eb },
3270	{ 0x0d, 0xffff, 0x1666 },
3271	{ 0x00, 0xffff, 0x10a3 },
3272	{ 0x06, 0xffff, 0xf050 },
3273	{ 0x04, 0x0000, 0x0010 },
3274	{ 0x1d, 0x4000, 0x0000 },
3275	};
3276
3277	rtl_hw_start_8168g(tp);
3278	rtl_ephy_init(tp, e_info_8168g_2);
3279	}
3280
3281	static void rtl8411b_fix_phy_down(struct rtl8169_private *tp)
3282	{
3283	static const u16 fix_data[] = {
3284	/* 0xf800 */ 0xe008, 0xe00a, 0xe00c, 0xe00e, 0xe027, 0xe04f, 0xe05e, 0xe065,
3285	/* 0xf810 */ 0xc602, 0xbe00, 0x0000, 0xc502, 0xbd00, 0x074c, 0xc302, 0xbb00,
3286	/* 0xf820 */ 0x080a, 0x6420, 0x48c2, 0x8c20, 0xc516, 0x64a4, 0x49c0, 0xf009,
3287	/* 0xf830 */ 0x74a2, 0x8ca5, 0x74a0, 0xc50e, 0x9ca2, 0x1c11, 0x9ca0, 0xe006,
3288	/* 0xf840 */ 0x74f8, 0x48c4, 0x8cf8, 0xc404, 0xbc00, 0xc403, 0xbc00, 0x0bf2,
3289	/* 0xf850 */ 0x0c0a, 0xe434, 0xd3c0, 0x49d9, 0xf01f, 0xc526, 0x64a5, 0x1400,
3290	/* 0xf860 */ 0xf007, 0x0c01, 0x8ca5, 0x1c15, 0xc51b, 0x9ca0, 0xe013, 0xc519,
3291	/* 0xf870 */ 0x74a0, 0x48c4, 0x8ca0, 0xc516, 0x74a4, 0x48c8, 0x48ca, 0x9ca4,
3292	/* 0xf880 */ 0xc512, 0x1b00, 0x9ba0, 0x1b1c, 0x483f, 0x9ba2, 0x1b04, 0xc508,
3293	/* 0xf890 */ 0x9ba0, 0xc505, 0xbd00, 0xc502, 0xbd00, 0x0300, 0x051e, 0xe434,
3294	/* 0xf8a0 */ 0xe018, 0xe092, 0xde20, 0xd3c0, 0xc50f, 0x76a4, 0x49e3, 0xf007,
3295	/* 0xf8b0 */ 0x49c0, 0xf103, 0xc607, 0xbe00, 0xc606, 0xbe00, 0xc602, 0xbe00,
3296	/* 0xf8c0 */ 0x0c4c, 0x0c28, 0x0c2c, 0xdc00, 0xc707, 0x1d00, 0x8de2, 0x48c1,
3297	/* 0xf8d0 */ 0xc502, 0xbd00, 0x00aa, 0xe0c0, 0xc502, 0xbd00, 0x0132
3298	};
3299	unsigned long flags;
3300	int i;
3301
3302	raw_spin_lock_irqsave(&tp->mac_ocp_lock, flags);
3303	for (i = 0; i < ARRAY_SIZE(fix_data); i++)
3304	__r8168_mac_ocp_write(tp, reg: 0xf800 + 2 * i, data: fix_data[i]);
3305	raw_spin_unlock_irqrestore(&tp->mac_ocp_lock, flags);
3306	}
3307
3308	static void rtl_hw_start_8411_2(struct rtl8169_private *tp)
3309	{
3310	static const struct ephy_info e_info_8411_2[] = {
3311	{ 0x00, 0x0008, 0x0000 },
3312	{ 0x0c, 0x37d0, 0x0820 },
3313	{ 0x1e, 0x0000, 0x0001 },
3314	{ 0x19, 0x8021, 0x0000 },
3315	{ 0x1e, 0x0000, 0x2000 },
3316	{ 0x0d, 0x0100, 0x0200 },
3317	{ 0x00, 0x0000, 0x0080 },
3318	{ 0x06, 0x0000, 0x0010 },
3319	{ 0x04, 0x0000, 0x0010 },
3320	{ 0x1d, 0x0000, 0x4000 },
3321	};
3322
3323	rtl_hw_start_8168g(tp);
3324
3325	rtl_ephy_init(tp, e_info_8411_2);
3326
3327	/* The following Realtek-provided magic fixes an issue with the RX unit
3328	* getting confused after the PHY having been powered-down.
3329	*/
3330	r8168_mac_ocp_write(tp, reg: 0xFC28, data: 0x0000);
3331	r8168_mac_ocp_write(tp, reg: 0xFC2A, data: 0x0000);
3332	r8168_mac_ocp_write(tp, reg: 0xFC2C, data: 0x0000);
3333	r8168_mac_ocp_write(tp, reg: 0xFC2E, data: 0x0000);
3334	r8168_mac_ocp_write(tp, reg: 0xFC30, data: 0x0000);
3335	r8168_mac_ocp_write(tp, reg: 0xFC32, data: 0x0000);
3336	r8168_mac_ocp_write(tp, reg: 0xFC34, data: 0x0000);
3337	r8168_mac_ocp_write(tp, reg: 0xFC36, data: 0x0000);
3338	mdelay(3);
3339	r8168_mac_ocp_write(tp, reg: 0xFC26, data: 0x0000);
3340
3341	rtl8411b_fix_phy_down(tp);
3342
3343	r8168_mac_ocp_write(tp, reg: 0xFC26, data: 0x8000);
3344
3345	r8168_mac_ocp_write(tp, reg: 0xFC2A, data: 0x0743);
3346	r8168_mac_ocp_write(tp, reg: 0xFC2C, data: 0x0801);
3347	r8168_mac_ocp_write(tp, reg: 0xFC2E, data: 0x0BE9);
3348	r8168_mac_ocp_write(tp, reg: 0xFC30, data: 0x02FD);
3349	r8168_mac_ocp_write(tp, reg: 0xFC32, data: 0x0C25);
3350	r8168_mac_ocp_write(tp, reg: 0xFC34, data: 0x00A9);
3351	r8168_mac_ocp_write(tp, reg: 0xFC36, data: 0x012D);
3352	}
3353
3354	static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
3355	{
3356	static const struct ephy_info e_info_8168h_1[] = {
3357	{ 0x1e, 0x0800, 0x0001 },
3358	{ 0x1d, 0x0000, 0x0800 },
3359	{ 0x05, 0xffff, 0x2089 },
3360	{ 0x06, 0xffff, 0x5881 },
3361	{ 0x04, 0xffff, 0x854a },
3362	{ 0x01, 0xffff, 0x068b }
3363	};
3364	int rg_saw_cnt;
3365
3366	rtl_ephy_init(tp, e_info_8168h_1);
3367
3368	rtl_set_fifo_size(tp, rx_stat: 0x08, tx_stat: 0x10, rx_dyn: 0x02, tx_dyn: 0x06);
3369	rtl8168g_set_pause_thresholds(tp, low: 0x38, high: 0x48);
3370
3371	rtl_set_def_aspm_entry_latency(tp);
3372
3373	rtl_reset_packet_filter(tp);
3374
3375	rtl_eri_set_bits(tp, addr: 0xdc, p: 0x001c);
3376
3377	rtl_eri_write(tp, addr: 0x5f0, ERIAR_MASK_0011, val: 0x4f87);
3378
3379	rtl_disable_rxdvgate(tp);
3380
3381	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3382	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_0011, val: 0x0000);
3383
3384	rtl8168_config_eee_mac(tp);
3385
3386	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3387	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3388
3389	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3390
3391	rtl_eri_clear_bits(tp, addr: 0x1b0, BIT(12));
3392
3393	rtl_pcie_state_l2l3_disable(tp);
3394
3395	rg_saw_cnt = phy_read_paged(phydev: tp->phydev, page: 0x0c42, regnum: 0x13) & 0x3fff;
3396	if (rg_saw_cnt > 0) {
3397	u16 sw_cnt_1ms_ini;
3398
3399	sw_cnt_1ms_ini = 16000000/rg_saw_cnt;
3400	sw_cnt_1ms_ini &= 0x0fff;
3401	r8168_mac_ocp_modify(tp, reg: 0xd412, mask: 0x0fff, set: sw_cnt_1ms_ini);
3402	}
3403
3404	r8168_mac_ocp_modify(tp, reg: 0xe056, mask: 0x00f0, set: 0x0070);
3405	r8168_mac_ocp_modify(tp, reg: 0xe052, mask: 0x6000, set: 0x8008);
3406	r8168_mac_ocp_modify(tp, reg: 0xe0d6, mask: 0x01ff, set: 0x017f);
3407	r8168_mac_ocp_modify(tp, reg: 0xd420, mask: 0x0fff, set: 0x047f);
3408
3409	r8168_mac_ocp_write(tp, reg: 0xe63e, data: 0x0001);
3410	r8168_mac_ocp_write(tp, reg: 0xe63e, data: 0x0000);
3411	r8168_mac_ocp_write(tp, reg: 0xc094, data: 0x0000);
3412	r8168_mac_ocp_write(tp, reg: 0xc09e, data: 0x0000);
3413	}
3414
3415	static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
3416	{
3417	rtl8168ep_stop_cmac(tp);
3418
3419	rtl_set_fifo_size(tp, rx_stat: 0x08, tx_stat: 0x10, rx_dyn: 0x02, tx_dyn: 0x06);
3420	rtl8168g_set_pause_thresholds(tp, low: 0x2f, high: 0x5f);
3421
3422	rtl_set_def_aspm_entry_latency(tp);
3423
3424	rtl_reset_packet_filter(tp);
3425
3426	rtl_eri_write(tp, addr: 0x5f0, ERIAR_MASK_0011, val: 0x4f87);
3427
3428	rtl_disable_rxdvgate(tp);
3429
3430	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3431	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_0011, val: 0x0000);
3432
3433	rtl8168_config_eee_mac(tp);
3434
3435	rtl_w0w1_eri(tp, addr: 0x2fc, p: 0x01, m: 0x06);
3436
3437	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3438
3439	rtl_pcie_state_l2l3_disable(tp);
3440	}
3441
3442	static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
3443	{
3444	static const struct ephy_info e_info_8168ep_3[] = {
3445	{ 0x00, 0x0000, 0x0080 },
3446	{ 0x0d, 0x0100, 0x0200 },
3447	{ 0x19, 0x8021, 0x0000 },
3448	{ 0x1e, 0x0000, 0x2000 },
3449	};
3450
3451	rtl_ephy_init(tp, e_info_8168ep_3);
3452
3453	rtl_hw_start_8168ep(tp);
3454
3455	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3456	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3457
3458	r8168_mac_ocp_modify(tp, reg: 0xd3e2, mask: 0x0fff, set: 0x0271);
3459	r8168_mac_ocp_modify(tp, reg: 0xd3e4, mask: 0x00ff, set: 0x0000);
3460	r8168_mac_ocp_modify(tp, reg: 0xe860, mask: 0x0000, set: 0x0080);
3461	}
3462
3463	static void rtl_hw_start_8117(struct rtl8169_private *tp)
3464	{
3465	static const struct ephy_info e_info_8117[] = {
3466	{ 0x19, 0x0040, 0x1100 },
3467	{ 0x59, 0x0040, 0x1100 },
3468	};
3469	int rg_saw_cnt;
3470
3471	rtl8168ep_stop_cmac(tp);
3472	rtl_ephy_init(tp, e_info_8117);
3473
3474	rtl_set_fifo_size(tp, rx_stat: 0x08, tx_stat: 0x10, rx_dyn: 0x02, tx_dyn: 0x06);
3475	rtl8168g_set_pause_thresholds(tp, low: 0x2f, high: 0x5f);
3476
3477	rtl_set_def_aspm_entry_latency(tp);
3478
3479	rtl_reset_packet_filter(tp);
3480
3481	rtl_eri_set_bits(tp, addr: 0xd4, p: 0x0010);
3482
3483	rtl_eri_write(tp, addr: 0x5f0, ERIAR_MASK_0011, val: 0x4f87);
3484
3485	rtl_disable_rxdvgate(tp);
3486
3487	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3488	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_0011, val: 0x0000);
3489
3490	rtl8168_config_eee_mac(tp);
3491
3492	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3493	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
3494
3495	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~TX_10M_PS_EN);
3496
3497	rtl_eri_clear_bits(tp, addr: 0x1b0, BIT(12));
3498
3499	rtl_pcie_state_l2l3_disable(tp);
3500
3501	rg_saw_cnt = phy_read_paged(phydev: tp->phydev, page: 0x0c42, regnum: 0x13) & 0x3fff;
3502	if (rg_saw_cnt > 0) {
3503	u16 sw_cnt_1ms_ini;
3504
3505	sw_cnt_1ms_ini = (16000000 / rg_saw_cnt) & 0x0fff;
3506	r8168_mac_ocp_modify(tp, reg: 0xd412, mask: 0x0fff, set: sw_cnt_1ms_ini);
3507	}
3508
3509	r8168_mac_ocp_modify(tp, reg: 0xe056, mask: 0x00f0, set: 0x0070);
3510	r8168_mac_ocp_write(tp, reg: 0xea80, data: 0x0003);
3511	r8168_mac_ocp_modify(tp, reg: 0xe052, mask: 0x0000, set: 0x0009);
3512	r8168_mac_ocp_modify(tp, reg: 0xd420, mask: 0x0fff, set: 0x047f);
3513
3514	r8168_mac_ocp_write(tp, reg: 0xe63e, data: 0x0001);
3515	r8168_mac_ocp_write(tp, reg: 0xe63e, data: 0x0000);
3516	r8168_mac_ocp_write(tp, reg: 0xc094, data: 0x0000);
3517	r8168_mac_ocp_write(tp, reg: 0xc09e, data: 0x0000);
3518
3519	/* firmware is for MAC only */
3520	r8169_apply_firmware(tp);
3521	}
3522
3523	static void rtl_hw_start_8102e_1(struct rtl8169_private *tp)
3524	{
3525	static const struct ephy_info e_info_8102e_1[] = {
3526	{ 0x01, 0, 0x6e65 },
3527	{ 0x02, 0, 0x091f },
3528	{ 0x03, 0, 0xc2f9 },
3529	{ 0x06, 0, 0xafb5 },
3530	{ 0x07, 0, 0x0e00 },
3531	{ 0x19, 0, 0xec80 },
3532	{ 0x01, 0, 0x2e65 },
3533	{ 0x01, 0, 0x6e65 }
3534	};
3535	u8 cfg1;
3536
3537	rtl_set_def_aspm_entry_latency(tp);
3538
3539	RTL_W8(tp, DBG_REG, FIX_NAK_1);
3540
3541	RTL_W8(tp, Config1,
3542	LEDS1 | LEDS0 | Speed_down | MEMMAP | IOMAP | VPD | PMEnable);
3543	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3544
3545	cfg1 = RTL_R8(tp, Config1);
3546	if ((cfg1 & LEDS0) && (cfg1 & LEDS1))
3547	RTL_W8(tp, Config1, cfg1 & ~LEDS0);
3548
3549	rtl_ephy_init(tp, e_info_8102e_1);
3550	}
3551
3552	static void rtl_hw_start_8102e_2(struct rtl8169_private *tp)
3553	{
3554	rtl_set_def_aspm_entry_latency(tp);
3555
3556	RTL_W8(tp, Config1, MEMMAP | IOMAP | VPD | PMEnable);
3557	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3558	}
3559
3560	static void rtl_hw_start_8102e_3(struct rtl8169_private *tp)
3561	{
3562	rtl_hw_start_8102e_2(tp);
3563
3564	rtl_ephy_write(tp, reg_addr: 0x03, value: 0xc2f9);
3565	}
3566
3567	static void rtl_hw_start_8401(struct rtl8169_private *tp)
3568	{
3569	static const struct ephy_info e_info_8401[] = {
3570	{ 0x01, 0xffff, 0x6fe5 },
3571	{ 0x03, 0xffff, 0x0599 },
3572	{ 0x06, 0xffff, 0xaf25 },
3573	{ 0x07, 0xffff, 0x8e68 },
3574	};
3575
3576	rtl_ephy_init(tp, e_info_8401);
3577	RTL_W8(tp, Config3, RTL_R8(tp, Config3) & ~Beacon_en);
3578	}
3579
3580	static void rtl_hw_start_8105e_1(struct rtl8169_private *tp)
3581	{
3582	static const struct ephy_info e_info_8105e_1[] = {
3583	{ 0x07, 0, 0x4000 },
3584	{ 0x19, 0, 0x0200 },
3585	{ 0x19, 0, 0x0020 },
3586	{ 0x1e, 0, 0x2000 },
3587	{ 0x03, 0, 0x0001 },
3588	{ 0x19, 0, 0x0100 },
3589	{ 0x19, 0, 0x0004 },
3590	{ 0x0a, 0, 0x0020 }
3591	};
3592
3593	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3594	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3595
3596	/* Disable Early Tally Counter */
3597	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) & ~0x010000);
3598
3599	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3600	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) | PFM_EN);
3601
3602	rtl_ephy_init(tp, e_info_8105e_1);
3603
3604	rtl_pcie_state_l2l3_disable(tp);
3605	}
3606
3607	static void rtl_hw_start_8105e_2(struct rtl8169_private *tp)
3608	{
3609	rtl_hw_start_8105e_1(tp);
3610	rtl_ephy_write(tp, reg_addr: 0x1e, value: rtl_ephy_read(tp, reg_addr: 0x1e) | 0x8000);
3611	}
3612
3613	static void rtl_hw_start_8402(struct rtl8169_private *tp)
3614	{
3615	static const struct ephy_info e_info_8402[] = {
3616	{ 0x19, 0xffff, 0xff64 },
3617	{ 0x1e, 0, 0x4000 }
3618	};
3619
3620	rtl_set_def_aspm_entry_latency(tp);
3621
3622	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3623	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3624
3625	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
3626
3627	rtl_ephy_init(tp, e_info_8402);
3628
3629	rtl_set_fifo_size(tp, rx_stat: 0x00, tx_stat: 0x00, rx_dyn: 0x02, tx_dyn: 0x06);
3630	rtl_reset_packet_filter(tp);
3631	rtl_eri_write(tp, addr: 0xc0, ERIAR_MASK_0011, val: 0x0000);
3632	rtl_eri_write(tp, addr: 0xb8, ERIAR_MASK_0011, val: 0x0000);
3633	rtl_w0w1_eri(tp, addr: 0x0d4, p: 0x0e00, m: 0xff00);
3634
3635	/* disable EEE */
3636	rtl_eri_write(tp, addr: 0x1b0, ERIAR_MASK_0011, val: 0x0000);
3637
3638	rtl_pcie_state_l2l3_disable(tp);
3639	}
3640
3641	static void rtl_hw_start_8106(struct rtl8169_private *tp)
3642	{
3643	/* Force LAN exit from ASPM if Rx/Tx are not idle */
3644	RTL_W32(tp, FuncEvent, RTL_R32(tp, FuncEvent) | 0x002800);
3645
3646	RTL_W32(tp, MISC, (RTL_R32(tp, MISC) | DISABLE_LAN_EN) & ~EARLY_TALLY_EN);
3647	RTL_W8(tp, MCU, RTL_R8(tp, MCU) | EN_NDP | EN_OOB_RESET);
3648	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
3649
3650	/* L0 7us, L1 32us - needed to avoid issues with link-up detection */
3651	rtl_set_aspm_entry_latency(tp, val: 0x2f);
3652
3653	rtl_eri_write(tp, addr: 0x1d0, ERIAR_MASK_0011, val: 0x0000);
3654
3655	/* disable EEE */
3656	rtl_eri_write(tp, addr: 0x1b0, ERIAR_MASK_0011, val: 0x0000);
3657
3658	rtl_pcie_state_l2l3_disable(tp);
3659	}
3660
3661	DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond)
3662	{
3663	return r8168_mac_ocp_read(tp, reg: 0xe00e) & BIT(13);
3664	}
3665
3666	static void rtl_hw_start_8125_common(struct rtl8169_private *tp)
3667	{
3668	rtl_pcie_state_l2l3_disable(tp);
3669
3670	RTL_W16(tp, 0x382, 0x221b);
3671	RTL_W8(tp, 0x4500, 0);
3672	RTL_W16(tp, 0x4800, 0);
3673
3674	/* disable UPS */
3675	r8168_mac_ocp_modify(tp, reg: 0xd40a, mask: 0x0010, set: 0x0000);
3676
3677	RTL_W8(tp, Config1, RTL_R8(tp, Config1) & ~0x10);
3678
3679	r8168_mac_ocp_write(tp, reg: 0xc140, data: 0xffff);
3680	r8168_mac_ocp_write(tp, reg: 0xc142, data: 0xffff);
3681
3682	r8168_mac_ocp_modify(tp, reg: 0xd3e2, mask: 0x0fff, set: 0x03a9);
3683	r8168_mac_ocp_modify(tp, reg: 0xd3e4, mask: 0x00ff, set: 0x0000);
3684	r8168_mac_ocp_modify(tp, reg: 0xe860, mask: 0x0000, set: 0x0080);
3685
3686	/* disable new tx descriptor format */
3687	r8168_mac_ocp_modify(tp, reg: 0xeb58, mask: 0x0001, set: 0x0000);
3688
3689	if (tp->mac_version == RTL_GIGA_MAC_VER_65)
3690	RTL_W8(tp, 0xD8, RTL_R8(tp, 0xD8) & ~0x02);
3691
3692	if (tp->mac_version == RTL_GIGA_MAC_VER_65)
3693	r8168_mac_ocp_modify(tp, reg: 0xe614, mask: 0x0700, set: 0x0400);
3694	else if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3695	r8168_mac_ocp_modify(tp, reg: 0xe614, mask: 0x0700, set: 0x0200);
3696	else
3697	r8168_mac_ocp_modify(tp, reg: 0xe614, mask: 0x0700, set: 0x0300);
3698
3699	if (tp->mac_version == RTL_GIGA_MAC_VER_63)
3700	r8168_mac_ocp_modify(tp, reg: 0xe63e, mask: 0x0c30, set: 0x0000);
3701	else
3702	r8168_mac_ocp_modify(tp, reg: 0xe63e, mask: 0x0c30, set: 0x0020);
3703
3704	r8168_mac_ocp_modify(tp, reg: 0xc0b4, mask: 0x0000, set: 0x000c);
3705	r8168_mac_ocp_modify(tp, reg: 0xeb6a, mask: 0x00ff, set: 0x0033);
3706	r8168_mac_ocp_modify(tp, reg: 0xeb50, mask: 0x03e0, set: 0x0040);
3707	r8168_mac_ocp_modify(tp, reg: 0xe056, mask: 0x00f0, set: 0x0030);
3708	r8168_mac_ocp_modify(tp, reg: 0xe040, mask: 0x1000, set: 0x0000);
3709	r8168_mac_ocp_modify(tp, reg: 0xea1c, mask: 0x0003, set: 0x0001);
3710	if (tp->mac_version == RTL_GIGA_MAC_VER_65)
3711	r8168_mac_ocp_modify(tp, reg: 0xea1c, mask: 0x0300, set: 0x0000);
3712	else
3713	r8168_mac_ocp_modify(tp, reg: 0xea1c, mask: 0x0004, set: 0x0000);
3714	r8168_mac_ocp_modify(tp, reg: 0xe0c0, mask: 0x4f0f, set: 0x4403);
3715	r8168_mac_ocp_modify(tp, reg: 0xe052, mask: 0x0080, set: 0x0068);
3716	r8168_mac_ocp_modify(tp, reg: 0xd430, mask: 0x0fff, set: 0x047f);
3717
3718	r8168_mac_ocp_modify(tp, reg: 0xea1c, mask: 0x0004, set: 0x0000);
3719	r8168_mac_ocp_modify(tp, reg: 0xeb54, mask: 0x0000, set: 0x0001);
3720	udelay(1);
3721	r8168_mac_ocp_modify(tp, reg: 0xeb54, mask: 0x0001, set: 0x0000);
3722	RTL_W16(tp, 0x1880, RTL_R16(tp, 0x1880) & ~0x0030);
3723
3724	r8168_mac_ocp_write(tp, reg: 0xe098, data: 0xc302);
3725
3726	rtl_loop_wait_low(tp, c: &rtl_mac_ocp_e00e_cond, d: 1000, n: 10);
3727
3728	if (tp->mac_version == RTL_GIGA_MAC_VER_61)
3729	rtl8125a_config_eee_mac(tp);
3730	else
3731	rtl8125b_config_eee_mac(tp);
3732
3733	rtl_disable_rxdvgate(tp);
3734	}
3735
3736	static void rtl_hw_start_8125a_2(struct rtl8169_private *tp)
3737	{
3738	static const struct ephy_info e_info_8125a_2[] = {
3739	{ 0x04, 0xffff, 0xd000 },
3740	{ 0x0a, 0xffff, 0x8653 },
3741	{ 0x23, 0xffff, 0xab66 },
3742	{ 0x20, 0xffff, 0x9455 },
3743	{ 0x21, 0xffff, 0x99ff },
3744	{ 0x29, 0xffff, 0xfe04 },
3745
3746	{ 0x44, 0xffff, 0xd000 },
3747	{ 0x4a, 0xffff, 0x8653 },
3748	{ 0x63, 0xffff, 0xab66 },
3749	{ 0x60, 0xffff, 0x9455 },
3750	{ 0x61, 0xffff, 0x99ff },
3751	{ 0x69, 0xffff, 0xfe04 },
3752	};
3753
3754	rtl_set_def_aspm_entry_latency(tp);
3755	rtl_ephy_init(tp, e_info_8125a_2);
3756	rtl_hw_start_8125_common(tp);
3757	}
3758
3759	static void rtl_hw_start_8125b(struct rtl8169_private *tp)
3760	{
3761	static const struct ephy_info e_info_8125b[] = {
3762	{ 0x0b, 0xffff, 0xa908 },
3763	{ 0x1e, 0xffff, 0x20eb },
3764	{ 0x4b, 0xffff, 0xa908 },
3765	{ 0x5e, 0xffff, 0x20eb },
3766	{ 0x22, 0x0030, 0x0020 },
3767	{ 0x62, 0x0030, 0x0020 },
3768	};
3769
3770	rtl_set_def_aspm_entry_latency(tp);
3771	rtl_ephy_init(tp, e_info_8125b);
3772	rtl_hw_start_8125_common(tp);
3773	}
3774
3775	static void rtl_hw_start_8126a(struct rtl8169_private *tp)
3776	{
3777	rtl_set_def_aspm_entry_latency(tp);
3778	rtl_hw_start_8125_common(tp);
3779	}
3780
3781	static void rtl_hw_config(struct rtl8169_private *tp)
3782	{
3783	static const rtl_generic_fct hw_configs[] = {
3784	[RTL_GIGA_MAC_VER_07] = rtl_hw_start_8102e_1,
3785	[RTL_GIGA_MAC_VER_08] = rtl_hw_start_8102e_3,
3786	[RTL_GIGA_MAC_VER_09] = rtl_hw_start_8102e_2,
3787	[RTL_GIGA_MAC_VER_10] = NULL,
3788	[RTL_GIGA_MAC_VER_11] = rtl_hw_start_8168b,
3789	[RTL_GIGA_MAC_VER_14] = rtl_hw_start_8401,
3790	[RTL_GIGA_MAC_VER_17] = rtl_hw_start_8168b,
3791	[RTL_GIGA_MAC_VER_18] = rtl_hw_start_8168cp_1,
3792	[RTL_GIGA_MAC_VER_19] = rtl_hw_start_8168c_1,
3793	[RTL_GIGA_MAC_VER_20] = rtl_hw_start_8168c_2,
3794	[RTL_GIGA_MAC_VER_21] = rtl_hw_start_8168c_2,
3795	[RTL_GIGA_MAC_VER_22] = rtl_hw_start_8168c_4,
3796	[RTL_GIGA_MAC_VER_23] = rtl_hw_start_8168cp_2,
3797	[RTL_GIGA_MAC_VER_24] = rtl_hw_start_8168cp_3,
3798	[RTL_GIGA_MAC_VER_25] = rtl_hw_start_8168d,
3799	[RTL_GIGA_MAC_VER_26] = rtl_hw_start_8168d,
3800	[RTL_GIGA_MAC_VER_28] = rtl_hw_start_8168d_4,
3801	[RTL_GIGA_MAC_VER_29] = rtl_hw_start_8105e_1,
3802	[RTL_GIGA_MAC_VER_30] = rtl_hw_start_8105e_2,
3803	[RTL_GIGA_MAC_VER_31] = rtl_hw_start_8168d,
3804	[RTL_GIGA_MAC_VER_32] = rtl_hw_start_8168e_1,
3805	[RTL_GIGA_MAC_VER_33] = rtl_hw_start_8168e_1,
3806	[RTL_GIGA_MAC_VER_34] = rtl_hw_start_8168e_2,
3807	[RTL_GIGA_MAC_VER_35] = rtl_hw_start_8168f_1,
3808	[RTL_GIGA_MAC_VER_36] = rtl_hw_start_8168f_1,
3809	[RTL_GIGA_MAC_VER_37] = rtl_hw_start_8402,
3810	[RTL_GIGA_MAC_VER_38] = rtl_hw_start_8411,
3811	[RTL_GIGA_MAC_VER_39] = rtl_hw_start_8106,
3812	[RTL_GIGA_MAC_VER_40] = rtl_hw_start_8168g_1,
3813	[RTL_GIGA_MAC_VER_42] = rtl_hw_start_8168g_2,
3814	[RTL_GIGA_MAC_VER_43] = rtl_hw_start_8168g_2,
3815	[RTL_GIGA_MAC_VER_44] = rtl_hw_start_8411_2,
3816	[RTL_GIGA_MAC_VER_46] = rtl_hw_start_8168h_1,
3817	[RTL_GIGA_MAC_VER_48] = rtl_hw_start_8168h_1,
3818	[RTL_GIGA_MAC_VER_51] = rtl_hw_start_8168ep_3,
3819	[RTL_GIGA_MAC_VER_52] = rtl_hw_start_8117,
3820	[RTL_GIGA_MAC_VER_53] = rtl_hw_start_8117,
3821	[RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2,
3822	[RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b,
3823	[RTL_GIGA_MAC_VER_65] = rtl_hw_start_8126a,
3824	};
3825
3826	if (hw_configs[tp->mac_version])
3827	hw_configs[tp->mac_version](tp);
3828	}
3829
3830	static void rtl_hw_start_8125(struct rtl8169_private *tp)
3831	{
3832	int i;
3833
3834	RTL_W8(tp, INT_CFG0_8125, 0x00);
3835
3836	/* disable interrupt coalescing */
3837	switch (tp->mac_version) {
3838	case RTL_GIGA_MAC_VER_61:
3839	for (i = 0xa00; i < 0xb00; i += 4)
3840	RTL_W32(tp, i, 0);
3841	break;
3842	case RTL_GIGA_MAC_VER_63:
3843	case RTL_GIGA_MAC_VER_65:
3844	for (i = 0xa00; i < 0xa80; i += 4)
3845	RTL_W32(tp, i, 0);
3846	RTL_W16(tp, INT_CFG1_8125, 0x0000);
3847	break;
3848	default:
3849	break;
3850	}
3851
3852	rtl_hw_config(tp);
3853	}
3854
3855	static void rtl_hw_start_8168(struct rtl8169_private *tp)
3856	{
3857	if (rtl_is_8168evl_up(tp))
3858	RTL_W8(tp, MaxTxPacketSize, EarlySize);
3859	else
3860	RTL_W8(tp, MaxTxPacketSize, TxPacketMax);
3861
3862	rtl_hw_config(tp);
3863
3864	/* disable interrupt coalescing */
3865	RTL_W16(tp, IntrMitigate, 0x0000);
3866	}
3867
3868	static void rtl_hw_start_8169(struct rtl8169_private *tp)
3869	{
3870	RTL_W8(tp, EarlyTxThres, NoEarlyTx);
3871
3872	tp->cp_cmd |= PCIMulRW;
3873
3874	if (tp->mac_version == RTL_GIGA_MAC_VER_02 ||
3875	tp->mac_version == RTL_GIGA_MAC_VER_03)
3876	tp->cp_cmd |= EnAnaPLL;
3877
3878	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3879
3880	rtl8169_set_magic_reg(tp);
3881
3882	/* disable interrupt coalescing */
3883	RTL_W16(tp, IntrMitigate, 0x0000);
3884	}
3885
3886	static void rtl_hw_start(struct rtl8169_private *tp)
3887	{
3888	rtl_unlock_config_regs(tp);
3889	/* disable aspm and clock request before ephy access */
3890	rtl_hw_aspm_clkreq_enable(tp, enable: false);
3891	RTL_W16(tp, CPlusCmd, tp->cp_cmd);
3892
3893	rtl_set_eee_txidle_timer(tp);
3894
3895	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
3896	rtl_hw_start_8169(tp);
3897	else if (rtl_is_8125(tp))
3898	rtl_hw_start_8125(tp);
3899	else
3900	rtl_hw_start_8168(tp);
3901
3902	rtl_enable_exit_l1(tp);
3903	rtl_hw_aspm_clkreq_enable(tp, enable: true);
3904	rtl_set_rx_max_size(tp);
3905	rtl_set_rx_tx_desc_registers(tp);
3906	rtl_lock_config_regs(tp);
3907
3908	rtl_jumbo_config(tp);
3909
3910	/* Initially a 10 us delay. Turned it into a PCI commit. - FR */
3911	rtl_pci_commit(tp);
3912
3913	RTL_W8(tp, ChipCmd, CmdTxEnb | CmdRxEnb);
3914	rtl_init_rxcfg(tp);
3915	rtl_set_tx_config_registers(tp);
3916	rtl_set_rx_config_features(tp, features: tp->dev->features);
3917	rtl_set_rx_mode(dev: tp->dev);
3918	rtl_irq_enable(tp);
3919	}
3920
3921	static int rtl8169_change_mtu(struct net_device *dev, int new_mtu)
3922	{
3923	struct rtl8169_private *tp = netdev_priv(dev);
3924
3925	dev->mtu = new_mtu;
3926	netdev_update_features(dev);
3927	rtl_jumbo_config(tp);
3928	rtl_set_eee_txidle_timer(tp);
3929
3930	return 0;
3931	}
3932
3933	static void rtl8169_mark_to_asic(struct RxDesc *desc)
3934	{
3935	u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
3936
3937	desc->opts2 = 0;
3938	/* Force memory writes to complete before releasing descriptor */
3939	dma_wmb();
3940	WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE));
3941	}
3942
3943	static struct page *rtl8169_alloc_rx_data(struct rtl8169_private *tp,
3944	struct RxDesc *desc)
3945	{
3946	struct device *d = tp_to_dev(tp);
3947	int node = dev_to_node(dev: d);
3948	dma_addr_t mapping;
3949	struct page *data;
3950
3951	data = alloc_pages_node(nid: node, GFP_KERNEL, order: get_order(R8169_RX_BUF_SIZE));
3952	if (!data)
3953	return NULL;
3954
3955	mapping = dma_map_page(d, data, 0, R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3956	if (unlikely(dma_mapping_error(d, mapping))) {
3957	netdev_err(dev: tp->dev, format: "Failed to map RX DMA!\n");
3958	__free_pages(page: data, order: get_order(R8169_RX_BUF_SIZE));
3959	return NULL;
3960	}
3961
3962	desc->addr = cpu_to_le64(mapping);
3963	rtl8169_mark_to_asic(desc);
3964
3965	return data;
3966	}
3967
3968	static void rtl8169_rx_clear(struct rtl8169_private *tp)
3969	{
3970	int i;
3971
3972	for (i = 0; i < NUM_RX_DESC && tp->Rx_databuff[i]; i++) {
3973	dma_unmap_page(tp_to_dev(tp),
3974	le64_to_cpu(tp->RxDescArray[i].addr),
3975	R8169_RX_BUF_SIZE, DMA_FROM_DEVICE);
3976	__free_pages(page: tp->Rx_databuff[i], order: get_order(R8169_RX_BUF_SIZE));
3977	tp->Rx_databuff[i] = NULL;
3978	tp->RxDescArray[i].addr = 0;
3979	tp->RxDescArray[i].opts1 = 0;
3980	}
3981	}
3982
3983	static int rtl8169_rx_fill(struct rtl8169_private *tp)
3984	{
3985	int i;
3986
3987	for (i = 0; i < NUM_RX_DESC; i++) {
3988	struct page *data;
3989
3990	data = rtl8169_alloc_rx_data(tp, desc: tp->RxDescArray + i);
3991	if (!data) {
3992	rtl8169_rx_clear(tp);
3993	return -ENOMEM;
3994	}
3995	tp->Rx_databuff[i] = data;
3996	}
3997
3998	/* mark as last descriptor in the ring */
3999	tp->RxDescArray[NUM_RX_DESC - 1].opts1 |= cpu_to_le32(RingEnd);
4000
4001	return 0;
4002	}
4003
4004	static int rtl8169_init_ring(struct rtl8169_private *tp)
4005	{
4006	rtl8169_init_ring_indexes(tp);
4007
4008	memset(tp->tx_skb, 0, sizeof(tp->tx_skb));
4009	memset(tp->Rx_databuff, 0, sizeof(tp->Rx_databuff));
4010
4011	return rtl8169_rx_fill(tp);
4012	}
4013
4014	static void rtl8169_unmap_tx_skb(struct rtl8169_private *tp, unsigned int entry)
4015	{
4016	struct ring_info *tx_skb = tp->tx_skb + entry;
4017	struct TxDesc *desc = tp->TxDescArray + entry;
4018
4019	dma_unmap_single(tp_to_dev(tp), le64_to_cpu(desc->addr), tx_skb->len,
4020	DMA_TO_DEVICE);
4021	memset(desc, 0, sizeof(*desc));
4022	memset(tx_skb, 0, sizeof(*tx_skb));
4023	}
4024
4025	static void rtl8169_tx_clear_range(struct rtl8169_private *tp, u32 start,
4026	unsigned int n)
4027	{
4028	unsigned int i;
4029
4030	for (i = 0; i < n; i++) {
4031	unsigned int entry = (start + i) % NUM_TX_DESC;
4032	struct ring_info *tx_skb = tp->tx_skb + entry;
4033	unsigned int len = tx_skb->len;
4034
4035	if (len) {
4036	struct sk_buff *skb = tx_skb->skb;
4037
4038	rtl8169_unmap_tx_skb(tp, entry);
4039	if (skb)
4040	dev_consume_skb_any(skb);
4041	}
4042	}
4043	}
4044
4045	static void rtl8169_tx_clear(struct rtl8169_private *tp)
4046	{
4047	rtl8169_tx_clear_range(tp, start: tp->dirty_tx, NUM_TX_DESC);
4048	netdev_reset_queue(dev_queue: tp->dev);
4049	}
4050
4051	static void rtl8169_cleanup(struct rtl8169_private *tp)
4052	{
4053	napi_disable(n: &tp->napi);
4054
4055	/* Give a racing hard_start_xmit a few cycles to complete. */
4056	synchronize_net();
4057
4058	/* Disable interrupts */
4059	rtl8169_irq_mask_and_ack(tp);
4060
4061	rtl_rx_close(tp);
4062
4063	switch (tp->mac_version) {
4064	case RTL_GIGA_MAC_VER_28:
4065	case RTL_GIGA_MAC_VER_31:
4066	rtl_loop_wait_low(tp, c: &rtl_npq_cond, d: 20, n: 2000);
4067	break;
4068	case RTL_GIGA_MAC_VER_34 ... RTL_GIGA_MAC_VER_38:
4069	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
4070	rtl_loop_wait_high(tp, c: &rtl_txcfg_empty_cond, d: 100, n: 666);
4071	break;
4072	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_65:
4073	rtl_enable_rxdvgate(tp);
4074	fsleep(usecs: 2000);
4075	break;
4076	default:
4077	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) | StopReq);
4078	fsleep(usecs: 100);
4079	break;
4080	}
4081
4082	rtl_hw_reset(tp);
4083
4084	rtl8169_tx_clear(tp);
4085	rtl8169_init_ring_indexes(tp);
4086	}
4087
4088	static void rtl_reset_work(struct rtl8169_private *tp)
4089	{
4090	int i;
4091
4092	netif_stop_queue(dev: tp->dev);
4093
4094	rtl8169_cleanup(tp);
4095
4096	for (i = 0; i < NUM_RX_DESC; i++)
4097	rtl8169_mark_to_asic(desc: tp->RxDescArray + i);
4098
4099	napi_enable(n: &tp->napi);
4100	rtl_hw_start(tp);
4101	}
4102
4103	static void rtl8169_tx_timeout(struct net_device *dev, unsigned int txqueue)
4104	{
4105	struct rtl8169_private *tp = netdev_priv(dev);
4106
4107	rtl_schedule_task(tp, flag: RTL_FLAG_TASK_TX_TIMEOUT);
4108	}
4109
4110	static int rtl8169_tx_map(struct rtl8169_private *tp, const u32 *opts, u32 len,
4111	void *addr, unsigned int entry, bool desc_own)
4112	{
4113	struct TxDesc *txd = tp->TxDescArray + entry;
4114	struct device *d = tp_to_dev(tp);
4115	dma_addr_t mapping;
4116	u32 opts1;
4117	int ret;
4118
4119	mapping = dma_map_single(d, addr, len, DMA_TO_DEVICE);
4120	ret = dma_mapping_error(dev: d, dma_addr: mapping);
4121	if (unlikely(ret)) {
4122	if (net_ratelimit())
4123	netdev_err(dev: tp->dev, format: "Failed to map TX data!\n");
4124	return ret;
4125	}
4126
4127	txd->addr = cpu_to_le64(mapping);
4128	txd->opts2 = cpu_to_le32(opts[1]);
4129
4130	opts1 = opts[0] | len;
4131	if (entry == NUM_TX_DESC - 1)
4132	opts1 |= RingEnd;
4133	if (desc_own)
4134	opts1 |= DescOwn;
4135	txd->opts1 = cpu_to_le32(opts1);
4136
4137	tp->tx_skb[entry].len = len;
4138
4139	return 0;
4140	}
4141
4142	static int rtl8169_xmit_frags(struct rtl8169_private *tp, struct sk_buff *skb,
4143	const u32 *opts, unsigned int entry)
4144	{
4145	struct skb_shared_info *info = skb_shinfo(skb);
4146	unsigned int cur_frag;
4147
4148	for (cur_frag = 0; cur_frag < info->nr_frags; cur_frag++) {
4149	const skb_frag_t *frag = info->frags + cur_frag;
4150	void *addr = skb_frag_address(frag);
4151	u32 len = skb_frag_size(frag);
4152
4153	entry = (entry + 1) % NUM_TX_DESC;
4154
4155	if (unlikely(rtl8169_tx_map(tp, opts, len, addr, entry, true)))
4156	goto err_out;
4157	}
4158
4159	return 0;
4160
4161	err_out:
4162	rtl8169_tx_clear_range(tp, start: tp->cur_tx + 1, n: cur_frag);
4163	return -EIO;
4164	}
4165
4166	static bool rtl_skb_is_udp(struct sk_buff *skb)
4167	{
4168	int no = skb_network_offset(skb);
4169	struct ipv6hdr *i6h, _i6h;
4170	struct iphdr *ih, _ih;
4171
4172	switch (vlan_get_protocol(skb)) {
4173	case htons(ETH_P_IP):
4174	ih = skb_header_pointer(skb, offset: no, len: sizeof(_ih), buffer: &_ih);
4175	return ih && ih->protocol == IPPROTO_UDP;
4176	case htons(ETH_P_IPV6):
4177	i6h = skb_header_pointer(skb, offset: no, len: sizeof(_i6h), buffer: &_i6h);
4178	return i6h && i6h->nexthdr == IPPROTO_UDP;
4179	default:
4180	return false;
4181	}
4182	}
4183
4184	#define RTL_MIN_PATCH_LEN 47
4185
4186	/* see rtl8125_get_patch_pad_len() in r8125 vendor driver */
4187	static unsigned int rtl8125_quirk_udp_padto(struct rtl8169_private *tp,
4188	struct sk_buff *skb)
4189	{
4190	unsigned int padto = 0, len = skb->len;
4191
4192	if (rtl_is_8125(tp) && len < 128 + RTL_MIN_PATCH_LEN &&
4193	rtl_skb_is_udp(skb) && skb_transport_header_was_set(skb)) {
4194	unsigned int trans_data_len = skb_tail_pointer(skb) -
4195	skb_transport_header(skb);
4196
4197	if (trans_data_len >= offsetof(struct udphdr, len) &&
4198	trans_data_len < RTL_MIN_PATCH_LEN) {
4199	u16 dest = ntohs(udp_hdr(skb)->dest);
4200
4201	/* dest is a standard PTP port */
4202	if (dest == 319 || dest == 320)
4203	padto = len + RTL_MIN_PATCH_LEN - trans_data_len;
4204	}
4205
4206	if (trans_data_len < sizeof(struct udphdr))
4207	padto = max_t(unsigned int, padto,
4208	len + sizeof(struct udphdr) - trans_data_len);
4209	}
4210
4211	return padto;
4212	}
4213
4214	static unsigned int rtl_quirk_packet_padto(struct rtl8169_private *tp,
4215	struct sk_buff *skb)
4216	{
4217	unsigned int padto;
4218
4219	padto = rtl8125_quirk_udp_padto(tp, skb);
4220
4221	switch (tp->mac_version) {
4222	case RTL_GIGA_MAC_VER_34:
4223	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_65:
4224	padto = max_t(unsigned int, padto, ETH_ZLEN);
4225	break;
4226	default:
4227	break;
4228	}
4229
4230	return padto;
4231	}
4232
4233	static void rtl8169_tso_csum_v1(struct sk_buff *skb, u32 *opts)
4234	{
4235	u32 mss = skb_shinfo(skb)->gso_size;
4236
4237	if (mss) {
4238	opts[0] |= TD_LSO;
4239	opts[0] |= mss << TD0_MSS_SHIFT;
4240	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4241	const struct iphdr *ip = ip_hdr(skb);
4242
4243	if (ip->protocol == IPPROTO_TCP)
4244	opts[0] |= TD0_IP_CS | TD0_TCP_CS;
4245	else if (ip->protocol == IPPROTO_UDP)
4246	opts[0] |= TD0_IP_CS | TD0_UDP_CS;
4247	else
4248	WARN_ON_ONCE(1);
4249	}
4250	}
4251
4252	static bool rtl8169_tso_csum_v2(struct rtl8169_private *tp,
4253	struct sk_buff *skb, u32 *opts)
4254	{
4255	struct skb_shared_info *shinfo = skb_shinfo(skb);
4256	u32 mss = shinfo->gso_size;
4257
4258	if (mss) {
4259	if (shinfo->gso_type & SKB_GSO_TCPV4) {
4260	opts[0] |= TD1_GTSENV4;
4261	} else if (shinfo->gso_type & SKB_GSO_TCPV6) {
4262	if (skb_cow_head(skb, headroom: 0))
4263	return false;
4264
4265	tcp_v6_gso_csum_prep(skb);
4266	opts[0] |= TD1_GTSENV6;
4267	} else {
4268	WARN_ON_ONCE(1);
4269	}
4270
4271	opts[0] |= skb_transport_offset(skb) << GTTCPHO_SHIFT;
4272	opts[1] |= mss << TD1_MSS_SHIFT;
4273	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4274	u8 ip_protocol;
4275
4276	switch (vlan_get_protocol(skb)) {
4277	case htons(ETH_P_IP):
4278	opts[1] |= TD1_IPv4_CS;
4279	ip_protocol = ip_hdr(skb)->protocol;
4280	break;
4281
4282	case htons(ETH_P_IPV6):
4283	opts[1] |= TD1_IPv6_CS;
4284	ip_protocol = ipv6_hdr(skb)->nexthdr;
4285	break;
4286
4287	default:
4288	ip_protocol = IPPROTO_RAW;
4289	break;
4290	}
4291
4292	if (ip_protocol == IPPROTO_TCP)
4293	opts[1] |= TD1_TCP_CS;
4294	else if (ip_protocol == IPPROTO_UDP)
4295	opts[1] |= TD1_UDP_CS;
4296	else
4297	WARN_ON_ONCE(1);
4298
4299	opts[1] |= skb_transport_offset(skb) << TCPHO_SHIFT;
4300	} else {
4301	unsigned int padto = rtl_quirk_packet_padto(tp, skb);
4302
4303	/* skb_padto would free the skb on error */
4304	return !__skb_put_padto(skb, len: padto, free_on_error: false);
4305	}
4306
4307	return true;
4308	}
4309
4310	static unsigned int rtl_tx_slots_avail(struct rtl8169_private *tp)
4311	{
4312	return READ_ONCE(tp->dirty_tx) + NUM_TX_DESC - READ_ONCE(tp->cur_tx);
4313	}
4314
4315	/* Versions RTL8102e and from RTL8168c onwards support csum_v2 */
4316	static bool rtl_chip_supports_csum_v2(struct rtl8169_private *tp)
4317	{
4318	switch (tp->mac_version) {
4319	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
4320	case RTL_GIGA_MAC_VER_10 ... RTL_GIGA_MAC_VER_17:
4321	return false;
4322	default:
4323	return true;
4324	}
4325	}
4326
4327	static void rtl8169_doorbell(struct rtl8169_private *tp)
4328	{
4329	if (rtl_is_8125(tp))
4330	RTL_W16(tp, TxPoll_8125, BIT(0));
4331	else
4332	RTL_W8(tp, TxPoll, NPQ);
4333	}
4334
4335	static netdev_tx_t rtl8169_start_xmit(struct sk_buff *skb,
4336	struct net_device *dev)
4337	{
4338	unsigned int frags = skb_shinfo(skb)->nr_frags;
4339	struct rtl8169_private *tp = netdev_priv(dev);
4340	unsigned int entry = tp->cur_tx % NUM_TX_DESC;
4341	struct TxDesc *txd_first, *txd_last;
4342	bool stop_queue, door_bell;
4343	u32 opts[2];
4344
4345	if (unlikely(!rtl_tx_slots_avail(tp))) {
4346	if (net_ratelimit())
4347	netdev_err(dev, format: "BUG! Tx Ring full when queue awake!\n");
4348	goto err_stop_0;
4349	}
4350
4351	opts[1] = rtl8169_tx_vlan_tag(skb);
4352	opts[0] = 0;
4353
4354	if (!rtl_chip_supports_csum_v2(tp))
4355	rtl8169_tso_csum_v1(skb, opts);
4356	else if (!rtl8169_tso_csum_v2(tp, skb, opts))
4357	goto err_dma_0;
4358
4359	if (unlikely(rtl8169_tx_map(tp, opts, skb_headlen(skb), skb->data,
4360	entry, false)))
4361	goto err_dma_0;
4362
4363	txd_first = tp->TxDescArray + entry;
4364
4365	if (frags) {
4366	if (rtl8169_xmit_frags(tp, skb, opts, entry))
4367	goto err_dma_1;
4368	entry = (entry + frags) % NUM_TX_DESC;
4369	}
4370
4371	txd_last = tp->TxDescArray + entry;
4372	txd_last->opts1 |= cpu_to_le32(LastFrag);
4373	tp->tx_skb[entry].skb = skb;
4374
4375	skb_tx_timestamp(skb);
4376
4377	/* Force memory writes to complete before releasing descriptor */
4378	dma_wmb();
4379
4380	door_bell = __netdev_sent_queue(dev, bytes: skb->len, xmit_more: netdev_xmit_more());
4381
4382	txd_first->opts1 |= cpu_to_le32(DescOwn | FirstFrag);
4383
4384	/* rtl_tx needs to see descriptor changes before updated tp->cur_tx */
4385	smp_wmb();
4386
4387	WRITE_ONCE(tp->cur_tx, tp->cur_tx + frags + 1);
4388
4389	stop_queue = !netif_subqueue_maybe_stop(dev, 0, rtl_tx_slots_avail(tp),
4390	R8169_TX_STOP_THRS,
4391	R8169_TX_START_THRS);
4392	if (door_bell || stop_queue)
4393	rtl8169_doorbell(tp);
4394
4395	return NETDEV_TX_OK;
4396
4397	err_dma_1:
4398	rtl8169_unmap_tx_skb(tp, entry);
4399	err_dma_0:
4400	dev_kfree_skb_any(skb);
4401	dev->stats.tx_dropped++;
4402	return NETDEV_TX_OK;
4403
4404	err_stop_0:
4405	netif_stop_queue(dev);
4406	dev->stats.tx_dropped++;
4407	return NETDEV_TX_BUSY;
4408	}
4409
4410	static unsigned int rtl_last_frag_len(struct sk_buff *skb)
4411	{
4412	struct skb_shared_info *info = skb_shinfo(skb);
4413	unsigned int nr_frags = info->nr_frags;
4414
4415	if (!nr_frags)
4416	return UINT_MAX;
4417
4418	return skb_frag_size(frag: info->frags + nr_frags - 1);
4419	}
4420
4421	/* Workaround for hw issues with TSO on RTL8168evl */
4422	static netdev_features_t rtl8168evl_fix_tso(struct sk_buff *skb,
4423	netdev_features_t features)
4424	{
4425	/* IPv4 header has options field */
4426	if (vlan_get_protocol(skb) == htons(ETH_P_IP) &&
4427	ip_hdrlen(skb) > sizeof(struct iphdr))
4428	features &= ~NETIF_F_ALL_TSO;
4429
4430	/* IPv4 TCP header has options field */
4431	else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4 &&
4432	tcp_hdrlen(skb) > sizeof(struct tcphdr))
4433	features &= ~NETIF_F_ALL_TSO;
4434
4435	else if (rtl_last_frag_len(skb) <= 6)
4436	features &= ~NETIF_F_ALL_TSO;
4437
4438	return features;
4439	}
4440
4441	static netdev_features_t rtl8169_features_check(struct sk_buff *skb,
4442	struct net_device *dev,
4443	netdev_features_t features)
4444	{
4445	struct rtl8169_private *tp = netdev_priv(dev);
4446
4447	if (skb_is_gso(skb)) {
4448	if (tp->mac_version == RTL_GIGA_MAC_VER_34)
4449	features = rtl8168evl_fix_tso(skb, features);
4450
4451	if (skb_transport_offset(skb) > GTTCPHO_MAX &&
4452	rtl_chip_supports_csum_v2(tp))
4453	features &= ~NETIF_F_ALL_TSO;
4454	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
4455	/* work around hw bug on some chip versions */
4456	if (skb->len < ETH_ZLEN)
4457	features &= ~NETIF_F_CSUM_MASK;
4458
4459	if (rtl_quirk_packet_padto(tp, skb))
4460	features &= ~NETIF_F_CSUM_MASK;
4461
4462	if (skb_transport_offset(skb) > TCPHO_MAX &&
4463	rtl_chip_supports_csum_v2(tp))
4464	features &= ~NETIF_F_CSUM_MASK;
4465	}
4466
4467	return vlan_features_check(skb, features);
4468	}
4469
4470	static void rtl8169_pcierr_interrupt(struct net_device *dev)
4471	{
4472	struct rtl8169_private *tp = netdev_priv(dev);
4473	struct pci_dev *pdev = tp->pci_dev;
4474	int pci_status_errs;
4475	u16 pci_cmd;
4476
4477	pci_read_config_word(dev: pdev, PCI_COMMAND, val: &pci_cmd);
4478
4479	pci_status_errs = pci_status_get_and_clear_errors(pdev);
4480
4481	if (net_ratelimit())
4482	netdev_err(dev, format: "PCI error (cmd = 0x%04x, status_errs = 0x%04x)\n",
4483	pci_cmd, pci_status_errs);
4484
4485	rtl_schedule_task(tp, flag: RTL_FLAG_TASK_RESET_PENDING);
4486	}
4487
4488	static void rtl_tx(struct net_device *dev, struct rtl8169_private *tp,
4489	int budget)
4490	{
4491	unsigned int dirty_tx, bytes_compl = 0, pkts_compl = 0;
4492	struct sk_buff *skb;
4493
4494	dirty_tx = tp->dirty_tx;
4495
4496	while (READ_ONCE(tp->cur_tx) != dirty_tx) {
4497	unsigned int entry = dirty_tx % NUM_TX_DESC;
4498	u32 status;
4499
4500	status = le32_to_cpu(READ_ONCE(tp->TxDescArray[entry].opts1));
4501	if (status & DescOwn)
4502	break;
4503
4504	skb = tp->tx_skb[entry].skb;
4505	rtl8169_unmap_tx_skb(tp, entry);
4506
4507	if (skb) {
4508	pkts_compl++;
4509	bytes_compl += skb->len;
4510	napi_consume_skb(skb, budget);
4511	}
4512	dirty_tx++;
4513	}
4514
4515	if (tp->dirty_tx != dirty_tx) {
4516	dev_sw_netstats_tx_add(dev, packets: pkts_compl, len: bytes_compl);
4517	WRITE_ONCE(tp->dirty_tx, dirty_tx);
4518
4519	netif_subqueue_completed_wake(dev, 0, pkts_compl, bytes_compl,
4520	rtl_tx_slots_avail(tp),
4521	R8169_TX_START_THRS);
4522	/*
4523	* 8168 hack: TxPoll requests are lost when the Tx packets are
4524	* too close. Let's kick an extra TxPoll request when a burst
4525	* of start_xmit activity is detected (if it is not detected,
4526	* it is slow enough). -- FR
4527	* If skb is NULL then we come here again once a tx irq is
4528	* triggered after the last fragment is marked transmitted.
4529	*/
4530	if (READ_ONCE(tp->cur_tx) != dirty_tx && skb)
4531	rtl8169_doorbell(tp);
4532	}
4533	}
4534
4535	static inline int rtl8169_fragmented_frame(u32 status)
4536	{
4537	return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag);
4538	}
4539
4540	static inline void rtl8169_rx_csum(struct sk_buff *skb, u32 opts1)
4541	{
4542	u32 status = opts1 & (RxProtoMask | RxCSFailMask);
4543
4544	if (status == RxProtoTCP || status == RxProtoUDP)
4545	skb->ip_summed = CHECKSUM_UNNECESSARY;
4546	else
4547	skb_checksum_none_assert(skb);
4548	}
4549
4550	static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget)
4551	{
4552	struct device *d = tp_to_dev(tp);
4553	int count;
4554
4555	for (count = 0; count < budget; count++, tp->cur_rx++) {
4556	unsigned int pkt_size, entry = tp->cur_rx % NUM_RX_DESC;
4557	struct RxDesc *desc = tp->RxDescArray + entry;
4558	struct sk_buff *skb;
4559	const void *rx_buf;
4560	dma_addr_t addr;
4561	u32 status;
4562
4563	status = le32_to_cpu(READ_ONCE(desc->opts1));
4564	if (status & DescOwn)
4565	break;
4566
4567	/* This barrier is needed to keep us from reading
4568	* any other fields out of the Rx descriptor until
4569	* we know the status of DescOwn
4570	*/
4571	dma_rmb();
4572
4573	if (unlikely(status & RxRES)) {
4574	if (net_ratelimit())
4575	netdev_warn(dev, format: "Rx ERROR. status = %08x\n",
4576	status);
4577	dev->stats.rx_errors++;
4578	if (status & (RxRWT | RxRUNT))
4579	dev->stats.rx_length_errors++;
4580	if (status & RxCRC)
4581	dev->stats.rx_crc_errors++;
4582
4583	if (!(dev->features & NETIF_F_RXALL))
4584	goto release_descriptor;
4585	else if (status & RxRWT || !(status & (RxRUNT | RxCRC)))
4586	goto release_descriptor;
4587	}
4588
4589	pkt_size = status & GENMASK(13, 0);
4590	if (likely(!(dev->features & NETIF_F_RXFCS)))
4591	pkt_size -= ETH_FCS_LEN;
4592
4593	/* The driver does not support incoming fragmented frames.
4594	* They are seen as a symptom of over-mtu sized frames.
4595	*/
4596	if (unlikely(rtl8169_fragmented_frame(status))) {
4597	dev->stats.rx_dropped++;
4598	dev->stats.rx_length_errors++;
4599	goto release_descriptor;
4600	}
4601
4602	skb = napi_alloc_skb(napi: &tp->napi, length: pkt_size);
4603	if (unlikely(!skb)) {
4604	dev->stats.rx_dropped++;
4605	goto release_descriptor;
4606	}
4607
4608	addr = le64_to_cpu(desc->addr);
4609	rx_buf = page_address(tp->Rx_databuff[entry]);
4610
4611	dma_sync_single_for_cpu(dev: d, addr, size: pkt_size, dir: DMA_FROM_DEVICE);
4612	prefetch(rx_buf);
4613	skb_copy_to_linear_data(skb, from: rx_buf, len: pkt_size);
4614	skb->tail += pkt_size;
4615	skb->len = pkt_size;
4616	dma_sync_single_for_device(dev: d, addr, size: pkt_size, dir: DMA_FROM_DEVICE);
4617
4618	rtl8169_rx_csum(skb, opts1: status);
4619	skb->protocol = eth_type_trans(skb, dev);
4620
4621	rtl8169_rx_vlan_tag(desc, skb);
4622
4623	if (skb->pkt_type == PACKET_MULTICAST)
4624	dev->stats.multicast++;
4625
4626	napi_gro_receive(napi: &tp->napi, skb);
4627
4628	dev_sw_netstats_rx_add(dev, len: pkt_size);
4629	release_descriptor:
4630	rtl8169_mark_to_asic(desc);
4631	}
4632
4633	return count;
4634	}
4635
4636	static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance)
4637	{
4638	struct rtl8169_private *tp = dev_instance;
4639	u32 status = rtl_get_events(tp);
4640
4641	if ((status & 0xffff) == 0xffff || !(status & tp->irq_mask))
4642	return IRQ_NONE;
4643
4644	if (unlikely(status & SYSErr)) {
4645	rtl8169_pcierr_interrupt(dev: tp->dev);
4646	goto out;
4647	}
4648
4649	if (status & LinkChg)
4650	phy_mac_interrupt(phydev: tp->phydev);
4651
4652	if (unlikely(status & RxFIFOOver &&
4653	tp->mac_version == RTL_GIGA_MAC_VER_11)) {
4654	netif_stop_queue(dev: tp->dev);
4655	rtl_schedule_task(tp, flag: RTL_FLAG_TASK_RESET_PENDING);
4656	}
4657
4658	if (napi_schedule_prep(n: &tp->napi)) {
4659	rtl_irq_disable(tp);
4660	__napi_schedule(n: &tp->napi);
4661	}
4662	out:
4663	rtl_ack_events(tp, bits: status);
4664
4665	return IRQ_HANDLED;
4666	}
4667
4668	static void rtl_task(struct work_struct *work)
4669	{
4670	struct rtl8169_private *tp =
4671	container_of(work, struct rtl8169_private, wk.work);
4672	int ret;
4673
4674	rtnl_lock();
4675
4676	if (!test_bit(RTL_FLAG_TASK_ENABLED, tp->wk.flags))
4677	goto out_unlock;
4678
4679	if (test_and_clear_bit(nr: RTL_FLAG_TASK_TX_TIMEOUT, addr: tp->wk.flags)) {
4680	/* if chip isn't accessible, reset bus to revive it */
4681	if (RTL_R32(tp, TxConfig) == ~0) {
4682	ret = pci_reset_bus(dev: tp->pci_dev);
4683	if (ret < 0) {
4684	netdev_err(dev: tp->dev, format: "Can't reset secondary PCI bus, detach NIC\n");
4685	netif_device_detach(dev: tp->dev);
4686	goto out_unlock;
4687	}
4688	}
4689
4690	/* ASPM compatibility issues are a typical reason for tx timeouts */
4691	ret = pci_disable_link_state(pdev: tp->pci_dev, PCIE_LINK_STATE_L1 |
4692	PCIE_LINK_STATE_L0S);
4693	if (!ret)
4694	netdev_warn_once(tp->dev, "ASPM disabled on Tx timeout\n");
4695	goto reset;
4696	}
4697
4698	if (test_and_clear_bit(nr: RTL_FLAG_TASK_RESET_PENDING, addr: tp->wk.flags)) {
4699	reset:
4700	rtl_reset_work(tp);
4701	netif_wake_queue(dev: tp->dev);
4702	} else if (test_and_clear_bit(nr: RTL_FLAG_TASK_RESET_NO_QUEUE_WAKE, addr: tp->wk.flags)) {
4703	rtl_reset_work(tp);
4704	}
4705	out_unlock:
4706	rtnl_unlock();
4707	}
4708
4709	static int rtl8169_poll(struct napi_struct *napi, int budget)
4710	{
4711	struct rtl8169_private *tp = container_of(napi, struct rtl8169_private, napi);
4712	struct net_device *dev = tp->dev;
4713	int work_done;
4714
4715	rtl_tx(dev, tp, budget);
4716
4717	work_done = rtl_rx(dev, tp, budget);
4718
4719	if (work_done < budget && napi_complete_done(n: napi, work_done))
4720	rtl_irq_enable(tp);
4721
4722	return work_done;
4723	}
4724
4725	static void r8169_phylink_handler(struct net_device *ndev)
4726	{
4727	struct rtl8169_private *tp = netdev_priv(dev: ndev);
4728	struct device *d = tp_to_dev(tp);
4729
4730	if (netif_carrier_ok(dev: ndev)) {
4731	rtl_link_chg_patch(tp);
4732	pm_request_resume(dev: d);
4733	netif_wake_queue(dev: tp->dev);
4734	} else {
4735	/* In few cases rx is broken after link-down otherwise */
4736	if (rtl_is_8125(tp))
4737	rtl_schedule_task(tp, flag: RTL_FLAG_TASK_RESET_NO_QUEUE_WAKE);
4738	pm_runtime_idle(dev: d);
4739	}
4740
4741	phy_print_status(phydev: tp->phydev);
4742	}
4743
4744	static int r8169_phy_connect(struct rtl8169_private *tp)
4745	{
4746	struct phy_device *phydev = tp->phydev;
4747	phy_interface_t phy_mode;
4748	int ret;
4749
4750	phy_mode = tp->supports_gmii ? PHY_INTERFACE_MODE_GMII :
4751	PHY_INTERFACE_MODE_MII;
4752
4753	ret = phy_connect_direct(dev: tp->dev, phydev, handler: r8169_phylink_handler,
4754	interface: phy_mode);
4755	if (ret)
4756	return ret;
4757
4758	if (!tp->supports_gmii)
4759	phy_set_max_speed(phydev, SPEED_100);
4760
4761	phy_attached_info(phydev);
4762
4763	return 0;
4764	}
4765
4766	static void rtl8169_down(struct rtl8169_private *tp)
4767	{
4768	/* Clear all task flags */
4769	bitmap_zero(dst: tp->wk.flags, nbits: RTL_FLAG_MAX);
4770
4771	phy_stop(phydev: tp->phydev);
4772
4773	rtl8169_update_counters(tp);
4774
4775	pci_clear_master(dev: tp->pci_dev);
4776	rtl_pci_commit(tp);
4777
4778	rtl8169_cleanup(tp);
4779	rtl_disable_exit_l1(tp);
4780	rtl_prepare_power_down(tp);
4781
4782	if (tp->dash_type != RTL_DASH_NONE)
4783	rtl8168_driver_stop(tp);
4784	}
4785
4786	static void rtl8169_up(struct rtl8169_private *tp)
4787	{
4788	if (tp->dash_type != RTL_DASH_NONE)
4789	rtl8168_driver_start(tp);
4790
4791	pci_set_master(dev: tp->pci_dev);
4792	phy_init_hw(phydev: tp->phydev);
4793	phy_resume(phydev: tp->phydev);
4794	rtl8169_init_phy(tp);
4795	napi_enable(n: &tp->napi);
4796	set_bit(nr: RTL_FLAG_TASK_ENABLED, addr: tp->wk.flags);
4797	rtl_reset_work(tp);
4798
4799	phy_start(phydev: tp->phydev);
4800	}
4801
4802	static int rtl8169_close(struct net_device *dev)
4803	{
4804	struct rtl8169_private *tp = netdev_priv(dev);
4805	struct pci_dev *pdev = tp->pci_dev;
4806
4807	pm_runtime_get_sync(dev: &pdev->dev);
4808
4809	netif_stop_queue(dev);
4810	rtl8169_down(tp);
4811	rtl8169_rx_clear(tp);
4812
4813	cancel_work(work: &tp->wk.work);
4814
4815	free_irq(tp->irq, tp);
4816
4817	phy_disconnect(phydev: tp->phydev);
4818
4819	dma_free_coherent(dev: &pdev->dev, R8169_RX_RING_BYTES, cpu_addr: tp->RxDescArray,
4820	dma_handle: tp->RxPhyAddr);
4821	dma_free_coherent(dev: &pdev->dev, R8169_TX_RING_BYTES, cpu_addr: tp->TxDescArray,
4822	dma_handle: tp->TxPhyAddr);
4823	tp->TxDescArray = NULL;
4824	tp->RxDescArray = NULL;
4825
4826	pm_runtime_put_sync(dev: &pdev->dev);
4827
4828	return 0;
4829	}
4830
4831	#ifdef CONFIG_NET_POLL_CONTROLLER
4832	static void rtl8169_netpoll(struct net_device *dev)
4833	{
4834	struct rtl8169_private *tp = netdev_priv(dev);
4835
4836	rtl8169_interrupt(irq: tp->irq, dev_instance: tp);
4837	}
4838	#endif
4839
4840	static int rtl_open(struct net_device *dev)
4841	{
4842	struct rtl8169_private *tp = netdev_priv(dev);
4843	struct pci_dev *pdev = tp->pci_dev;
4844	unsigned long irqflags;
4845	int retval = -ENOMEM;
4846
4847	pm_runtime_get_sync(dev: &pdev->dev);
4848
4849	/*
4850	* Rx and Tx descriptors needs 256 bytes alignment.
4851	* dma_alloc_coherent provides more.
4852	*/
4853	tp->TxDescArray = dma_alloc_coherent(dev: &pdev->dev, R8169_TX_RING_BYTES,
4854	dma_handle: &tp->TxPhyAddr, GFP_KERNEL);
4855	if (!tp->TxDescArray)
4856	goto out;
4857
4858	tp->RxDescArray = dma_alloc_coherent(dev: &pdev->dev, R8169_RX_RING_BYTES,
4859	dma_handle: &tp->RxPhyAddr, GFP_KERNEL);
4860	if (!tp->RxDescArray)
4861	goto err_free_tx_0;
4862
4863	retval = rtl8169_init_ring(tp);
4864	if (retval < 0)
4865	goto err_free_rx_1;
4866
4867	rtl_request_firmware(tp);
4868
4869	irqflags = pci_dev_msi_enabled(pci_dev: pdev) ? IRQF_NO_THREAD : IRQF_SHARED;
4870	retval = request_irq(irq: tp->irq, handler: rtl8169_interrupt, flags: irqflags, name: dev->name, dev: tp);
4871	if (retval < 0)
4872	goto err_release_fw_2;
4873
4874	retval = r8169_phy_connect(tp);
4875	if (retval)
4876	goto err_free_irq;
4877
4878	rtl8169_up(tp);
4879	rtl8169_init_counter_offsets(tp);
4880	netif_start_queue(dev);
4881	out:
4882	pm_runtime_put_sync(dev: &pdev->dev);
4883
4884	return retval;
4885
4886	err_free_irq:
4887	free_irq(tp->irq, tp);
4888	err_release_fw_2:
4889	rtl_release_firmware(tp);
4890	rtl8169_rx_clear(tp);
4891	err_free_rx_1:
4892	dma_free_coherent(dev: &pdev->dev, R8169_RX_RING_BYTES, cpu_addr: tp->RxDescArray,
4893	dma_handle: tp->RxPhyAddr);
4894	tp->RxDescArray = NULL;
4895	err_free_tx_0:
4896	dma_free_coherent(dev: &pdev->dev, R8169_TX_RING_BYTES, cpu_addr: tp->TxDescArray,
4897	dma_handle: tp->TxPhyAddr);
4898	tp->TxDescArray = NULL;
4899	goto out;
4900	}
4901
4902	static void
4903	rtl8169_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
4904	{
4905	struct rtl8169_private *tp = netdev_priv(dev);
4906	struct pci_dev *pdev = tp->pci_dev;
4907	struct rtl8169_counters *counters = tp->counters;
4908
4909	pm_runtime_get_noresume(dev: &pdev->dev);
4910
4911	netdev_stats_to_stats64(stats64: stats, netdev_stats: &dev->stats);
4912	dev_fetch_sw_netstats(s: stats, netstats: dev->tstats);
4913
4914	/*
4915	* Fetch additional counter values missing in stats collected by driver
4916	* from tally counters.
4917	*/
4918	if (pm_runtime_active(dev: &pdev->dev))
4919	rtl8169_update_counters(tp);
4920
4921	/*
4922	* Subtract values fetched during initalization.
4923	* See rtl8169_init_counter_offsets for a description why we do that.
4924	*/
4925	stats->tx_errors = le64_to_cpu(counters->tx_errors) -
4926	le64_to_cpu(tp->tc_offset.tx_errors);
4927	stats->collisions = le32_to_cpu(counters->tx_multi_collision) -
4928	le32_to_cpu(tp->tc_offset.tx_multi_collision);
4929	stats->tx_aborted_errors = le16_to_cpu(counters->tx_aborted) -
4930	le16_to_cpu(tp->tc_offset.tx_aborted);
4931	stats->rx_missed_errors = le16_to_cpu(counters->rx_missed) -
4932	le16_to_cpu(tp->tc_offset.rx_missed);
4933
4934	pm_runtime_put_noidle(dev: &pdev->dev);
4935	}
4936
4937	static void rtl8169_net_suspend(struct rtl8169_private *tp)
4938	{
4939	netif_device_detach(dev: tp->dev);
4940
4941	if (netif_running(dev: tp->dev))
4942	rtl8169_down(tp);
4943	}
4944
4945	static int rtl8169_runtime_resume(struct device *dev)
4946	{
4947	struct rtl8169_private *tp = dev_get_drvdata(dev);
4948
4949	rtl_rar_set(tp, addr: tp->dev->dev_addr);
4950	__rtl8169_set_wol(tp, wolopts: tp->saved_wolopts);
4951
4952	if (tp->TxDescArray)
4953	rtl8169_up(tp);
4954
4955	netif_device_attach(dev: tp->dev);
4956
4957	return 0;
4958	}
4959
4960	static int rtl8169_suspend(struct device *device)
4961	{
4962	struct rtl8169_private *tp = dev_get_drvdata(dev: device);
4963
4964	rtnl_lock();
4965	rtl8169_net_suspend(tp);
4966	if (!device_may_wakeup(dev: tp_to_dev(tp)))
4967	clk_disable_unprepare(clk: tp->clk);
4968	rtnl_unlock();
4969
4970	return 0;
4971	}
4972
4973	static int rtl8169_resume(struct device *device)
4974	{
4975	struct rtl8169_private *tp = dev_get_drvdata(dev: device);
4976
4977	if (!device_may_wakeup(dev: tp_to_dev(tp)))
4978	clk_prepare_enable(clk: tp->clk);
4979
4980	/* Reportedly at least Asus X453MA truncates packets otherwise */
4981	if (tp->mac_version == RTL_GIGA_MAC_VER_37)
4982	rtl_init_rxcfg(tp);
4983
4984	return rtl8169_runtime_resume(dev: device);
4985	}
4986
4987	static int rtl8169_runtime_suspend(struct device *device)
4988	{
4989	struct rtl8169_private *tp = dev_get_drvdata(dev: device);
4990
4991	if (!tp->TxDescArray) {
4992	netif_device_detach(dev: tp->dev);
4993	return 0;
4994	}
4995
4996	rtnl_lock();
4997	__rtl8169_set_wol(tp, WAKE_PHY);
4998	rtl8169_net_suspend(tp);
4999	rtnl_unlock();
5000
5001	return 0;
5002	}
5003
5004	static int rtl8169_runtime_idle(struct device *device)
5005	{
5006	struct rtl8169_private *tp = dev_get_drvdata(dev: device);
5007
5008	if (tp->dash_enabled)
5009	return -EBUSY;
5010
5011	if (!netif_running(dev: tp->dev) || !netif_carrier_ok(dev: tp->dev))
5012	pm_schedule_suspend(dev: device, delay: 10000);
5013
5014	return -EBUSY;
5015	}
5016
5017	static const struct dev_pm_ops rtl8169_pm_ops = {
5018	SYSTEM_SLEEP_PM_OPS(rtl8169_suspend, rtl8169_resume)
5019	RUNTIME_PM_OPS(rtl8169_runtime_suspend, rtl8169_runtime_resume,
5020	rtl8169_runtime_idle)
5021	};
5022
5023	static void rtl_shutdown(struct pci_dev *pdev)
5024	{
5025	struct rtl8169_private *tp = pci_get_drvdata(pdev);
5026
5027	rtnl_lock();
5028	rtl8169_net_suspend(tp);
5029	rtnl_unlock();
5030
5031	/* Restore original MAC address */
5032	rtl_rar_set(tp, addr: tp->dev->perm_addr);
5033
5034	if (system_state == SYSTEM_POWER_OFF && !tp->dash_enabled) {
5035	pci_wake_from_d3(dev: pdev, enable: tp->saved_wolopts);
5036	pci_set_power_state(dev: pdev, PCI_D3hot);
5037	}
5038	}
5039
5040	static void rtl_remove_one(struct pci_dev *pdev)
5041	{
5042	struct rtl8169_private *tp = pci_get_drvdata(pdev);
5043
5044	if (pci_dev_run_wake(dev: pdev))
5045	pm_runtime_get_noresume(dev: &pdev->dev);
5046
5047	cancel_work_sync(work: &tp->wk.work);
5048
5049	if (IS_ENABLED(CONFIG_R8169_LEDS))
5050	r8169_remove_leds(leds: tp->leds);
5051
5052	unregister_netdev(dev: tp->dev);
5053
5054	if (tp->dash_type != RTL_DASH_NONE)
5055	rtl8168_driver_stop(tp);
5056
5057	rtl_release_firmware(tp);
5058
5059	/* restore original MAC address */
5060	rtl_rar_set(tp, addr: tp->dev->perm_addr);
5061	}
5062
5063	static const struct net_device_ops rtl_netdev_ops = {
5064	.ndo_open = rtl_open,
5065	.ndo_stop = rtl8169_close,
5066	.ndo_get_stats64 = rtl8169_get_stats64,
5067	.ndo_start_xmit = rtl8169_start_xmit,
5068	.ndo_features_check = rtl8169_features_check,
5069	.ndo_tx_timeout = rtl8169_tx_timeout,
5070	.ndo_validate_addr = eth_validate_addr,
5071	.ndo_change_mtu = rtl8169_change_mtu,
5072	.ndo_fix_features = rtl8169_fix_features,
5073	.ndo_set_features = rtl8169_set_features,
5074	.ndo_set_mac_address = rtl_set_mac_address,
5075	.ndo_eth_ioctl = phy_do_ioctl_running,
5076	.ndo_set_rx_mode = rtl_set_rx_mode,
5077	#ifdef CONFIG_NET_POLL_CONTROLLER
5078	.ndo_poll_controller = rtl8169_netpoll,
5079	#endif
5080
5081	};
5082
5083	static void rtl_set_irq_mask(struct rtl8169_private *tp)
5084	{
5085	tp->irq_mask = RxOK | RxErr | TxOK | TxErr | LinkChg;
5086
5087	if (tp->mac_version <= RTL_GIGA_MAC_VER_06)
5088	tp->irq_mask |= SYSErr | RxOverflow | RxFIFOOver;
5089	else if (tp->mac_version == RTL_GIGA_MAC_VER_11)
5090	/* special workaround needed */
5091	tp->irq_mask |= RxFIFOOver;
5092	else
5093	tp->irq_mask |= RxOverflow;
5094	}
5095
5096	static int rtl_alloc_irq(struct rtl8169_private *tp)
5097	{
5098	unsigned int flags;
5099
5100	switch (tp->mac_version) {
5101	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
5102	rtl_unlock_config_regs(tp);
5103	RTL_W8(tp, Config2, RTL_R8(tp, Config2) & ~MSIEnable);
5104	rtl_lock_config_regs(tp);
5105	fallthrough;
5106	case RTL_GIGA_MAC_VER_07 ... RTL_GIGA_MAC_VER_17:
5107	flags = PCI_IRQ_LEGACY;
5108	break;
5109	default:
5110	flags = PCI_IRQ_ALL_TYPES;
5111	break;
5112	}
5113
5114	return pci_alloc_irq_vectors(dev: tp->pci_dev, min_vecs: 1, max_vecs: 1, flags);
5115	}
5116
5117	static void rtl_read_mac_address(struct rtl8169_private *tp,
5118	u8 mac_addr[ETH_ALEN])
5119	{
5120	/* Get MAC address */
5121	if (rtl_is_8168evl_up(tp) && tp->mac_version != RTL_GIGA_MAC_VER_34) {
5122	u32 value;
5123
5124	value = rtl_eri_read(tp, addr: 0xe0);
5125	put_unaligned_le32(val: value, p: mac_addr);
5126	value = rtl_eri_read(tp, addr: 0xe4);
5127	put_unaligned_le16(val: value, p: mac_addr + 4);
5128	} else if (rtl_is_8125(tp)) {
5129	rtl_read_mac_from_reg(tp, mac: mac_addr, reg: MAC0_BKP);
5130	}
5131	}
5132
5133	DECLARE_RTL_COND(rtl_link_list_ready_cond)
5134	{
5135	return RTL_R8(tp, MCU) & LINK_LIST_RDY;
5136	}
5137
5138	static void r8168g_wait_ll_share_fifo_ready(struct rtl8169_private *tp)
5139	{
5140	rtl_loop_wait_high(tp, c: &rtl_link_list_ready_cond, d: 100, n: 42);
5141	}
5142
5143	static int r8169_mdio_read_reg(struct mii_bus *mii_bus, int phyaddr, int phyreg)
5144	{
5145	struct rtl8169_private *tp = mii_bus->priv;
5146
5147	if (phyaddr > 0)
5148	return -ENODEV;
5149
5150	return rtl_readphy(tp, location: phyreg);
5151	}
5152
5153	static int r8169_mdio_write_reg(struct mii_bus *mii_bus, int phyaddr,
5154	int phyreg, u16 val)
5155	{
5156	struct rtl8169_private *tp = mii_bus->priv;
5157
5158	if (phyaddr > 0)
5159	return -ENODEV;
5160
5161	rtl_writephy(tp, location: phyreg, val);
5162
5163	return 0;
5164	}
5165
5166	static int r8169_mdio_register(struct rtl8169_private *tp)
5167	{
5168	struct pci_dev *pdev = tp->pci_dev;
5169	struct mii_bus *new_bus;
5170	int ret;
5171
5172	/* On some boards with this chip version the BIOS is buggy and misses
5173	* to reset the PHY page selector. This results in the PHY ID read
5174	* accessing registers on a different page, returning a more or
5175	* less random value. Fix this by resetting the page selector first.
5176	*/
5177	if (tp->mac_version == RTL_GIGA_MAC_VER_25 ||
5178	tp->mac_version == RTL_GIGA_MAC_VER_26)
5179	r8169_mdio_write(tp, reg: 0x1f, value: 0);
5180
5181	new_bus = devm_mdiobus_alloc(dev: &pdev->dev);
5182	if (!new_bus)
5183	return -ENOMEM;
5184
5185	new_bus->name = "r8169";
5186	new_bus->priv = tp;
5187	new_bus->parent = &pdev->dev;
5188	new_bus->irq[0] = PHY_MAC_INTERRUPT;
5189	snprintf(buf: new_bus->id, MII_BUS_ID_SIZE, fmt: "r8169-%x-%x",
5190	pci_domain_nr(bus: pdev->bus), pci_dev_id(dev: pdev));
5191
5192	new_bus->read = r8169_mdio_read_reg;
5193	new_bus->write = r8169_mdio_write_reg;
5194
5195	ret = devm_mdiobus_register(&pdev->dev, new_bus);
5196	if (ret)
5197	return ret;
5198
5199	tp->phydev = mdiobus_get_phy(bus: new_bus, addr: 0);
5200	if (!tp->phydev) {
5201	return -ENODEV;
5202	} else if (!tp->phydev->drv) {
5203	/* Most chip versions fail with the genphy driver.
5204	* Therefore ensure that the dedicated PHY driver is loaded.
5205	*/
5206	dev_err(&pdev->dev, "no dedicated PHY driver found for PHY ID 0x%08x, maybe realtek.ko needs to be added to initramfs?\n",
5207	tp->phydev->phy_id);
5208	return -EUNATCH;
5209	}
5210
5211	tp->phydev->mac_managed_pm = true;
5212	if (rtl_supports_eee(tp))
5213	phy_support_eee(phydev: tp->phydev);
5214	phy_support_asym_pause(phydev: tp->phydev);
5215
5216	/* PHY will be woken up in rtl_open() */
5217	phy_suspend(phydev: tp->phydev);
5218
5219	return 0;
5220	}
5221
5222	static void rtl_hw_init_8168g(struct rtl8169_private *tp)
5223	{
5224	rtl_enable_rxdvgate(tp);
5225
5226	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5227	msleep(msecs: 1);
5228	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5229
5230	r8168_mac_ocp_modify(tp, reg: 0xe8de, BIT(14), set: 0);
5231	r8168g_wait_ll_share_fifo_ready(tp);
5232
5233	r8168_mac_ocp_modify(tp, reg: 0xe8de, mask: 0, BIT(15));
5234	r8168g_wait_ll_share_fifo_ready(tp);
5235	}
5236
5237	static void rtl_hw_init_8125(struct rtl8169_private *tp)
5238	{
5239	rtl_enable_rxdvgate(tp);
5240
5241	RTL_W8(tp, ChipCmd, RTL_R8(tp, ChipCmd) & ~(CmdTxEnb | CmdRxEnb));
5242	msleep(msecs: 1);
5243	RTL_W8(tp, MCU, RTL_R8(tp, MCU) & ~NOW_IS_OOB);
5244
5245	r8168_mac_ocp_modify(tp, reg: 0xe8de, BIT(14), set: 0);
5246	r8168g_wait_ll_share_fifo_ready(tp);
5247
5248	r8168_mac_ocp_write(tp, reg: 0xc0aa, data: 0x07d0);
5249	r8168_mac_ocp_write(tp, reg: 0xc0a6, data: 0x0150);
5250	r8168_mac_ocp_write(tp, reg: 0xc01e, data: 0x5555);
5251	r8168g_wait_ll_share_fifo_ready(tp);
5252	}
5253
5254	static void rtl_hw_initialize(struct rtl8169_private *tp)
5255	{
5256	switch (tp->mac_version) {
5257	case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53:
5258	rtl8168ep_stop_cmac(tp);
5259	fallthrough;
5260	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48:
5261	rtl_hw_init_8168g(tp);
5262	break;
5263	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_65:
5264	rtl_hw_init_8125(tp);
5265	break;
5266	default:
5267	break;
5268	}
5269	}
5270
5271	static int rtl_jumbo_max(struct rtl8169_private *tp)
5272	{
5273	/* Non-GBit versions don't support jumbo frames */
5274	if (!tp->supports_gmii)
5275	return 0;
5276
5277	switch (tp->mac_version) {
5278	/* RTL8169 */
5279	case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06:
5280	return JUMBO_7K;
5281	/* RTL8168b */
5282	case RTL_GIGA_MAC_VER_11:
5283	case RTL_GIGA_MAC_VER_17:
5284	return JUMBO_4K;
5285	/* RTL8168c */
5286	case RTL_GIGA_MAC_VER_18 ... RTL_GIGA_MAC_VER_24:
5287	return JUMBO_6K;
5288	default:
5289	return JUMBO_9K;
5290	}
5291	}
5292
5293	static void rtl_init_mac_address(struct rtl8169_private *tp)
5294	{
5295	u8 mac_addr[ETH_ALEN] __aligned(2) = {};
5296	struct net_device *dev = tp->dev;
5297	int rc;
5298
5299	rc = eth_platform_get_mac_address(dev: tp_to_dev(tp), mac_addr);
5300	if (!rc)
5301	goto done;
5302
5303	rtl_read_mac_address(tp, mac_addr);
5304	if (is_valid_ether_addr(addr: mac_addr))
5305	goto done;
5306
5307	rtl_read_mac_from_reg(tp, mac: mac_addr, reg: MAC0);
5308	if (is_valid_ether_addr(addr: mac_addr))
5309	goto done;
5310
5311	eth_random_addr(addr: mac_addr);
5312	dev->addr_assign_type = NET_ADDR_RANDOM;
5313	dev_warn(tp_to_dev(tp), "can't read MAC address, setting random one\n");
5314	done:
5315	eth_hw_addr_set(dev, addr: mac_addr);
5316	rtl_rar_set(tp, addr: mac_addr);
5317	}
5318
5319	/* register is set if system vendor successfully tested ASPM 1.2 */
5320	static bool rtl_aspm_is_safe(struct rtl8169_private *tp)
5321	{
5322	if (tp->mac_version >= RTL_GIGA_MAC_VER_61 &&
5323	r8168_mac_ocp_read(tp, reg: 0xc0b2) & 0xf)
5324	return true;
5325
5326	return false;
5327	}
5328
5329	static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5330	{
5331	struct rtl8169_private *tp;
5332	int jumbo_max, region, rc;
5333	enum mac_version chipset;
5334	struct net_device *dev;
5335	u32 txconfig;
5336	u16 xid;
5337
5338	dev = devm_alloc_etherdev(&pdev->dev, sizeof (*tp));
5339	if (!dev)
5340	return -ENOMEM;
5341
5342	SET_NETDEV_DEV(dev, &pdev->dev);
5343	dev->netdev_ops = &rtl_netdev_ops;
5344	tp = netdev_priv(dev);
5345	tp->dev = dev;
5346	tp->pci_dev = pdev;
5347	tp->supports_gmii = ent->driver_data == RTL_CFG_NO_GBIT ? 0 : 1;
5348	tp->ocp_base = OCP_STD_PHY_BASE;
5349
5350	raw_spin_lock_init(&tp->cfg9346_usage_lock);
5351	raw_spin_lock_init(&tp->config25_lock);
5352	raw_spin_lock_init(&tp->mac_ocp_lock);
5353	mutex_init(&tp->led_lock);
5354
5355	/* Get the *optional* external "ether_clk" used on some boards */
5356	tp->clk = devm_clk_get_optional_enabled(dev: &pdev->dev, id: "ether_clk");
5357	if (IS_ERR(ptr: tp->clk))
5358	return dev_err_probe(dev: &pdev->dev, err: PTR_ERR(ptr: tp->clk), fmt: "failed to get ether_clk\n");
5359
5360	/* enable device (incl. PCI PM wakeup and hotplug setup) */
5361	rc = pcim_enable_device(pdev);
5362	if (rc < 0)
5363	return dev_err_probe(dev: &pdev->dev, err: rc, fmt: "enable failure\n");
5364
5365	if (pcim_set_mwi(dev: pdev) < 0)
5366	dev_info(&pdev->dev, "Mem-Wr-Inval unavailable\n");
5367
5368	/* use first MMIO region */
5369	region = ffs(pci_select_bars(pdev, IORESOURCE_MEM)) - 1;
5370	if (region < 0)
5371	return dev_err_probe(dev: &pdev->dev, err: -ENODEV, fmt: "no MMIO resource found\n");
5372
5373	rc = pcim_iomap_regions(pdev, BIT(region), KBUILD_MODNAME);
5374	if (rc < 0)
5375	return dev_err_probe(dev: &pdev->dev, err: rc, fmt: "cannot remap MMIO, aborting\n");
5376
5377	tp->mmio_addr = pcim_iomap_table(pdev)[region];
5378
5379	txconfig = RTL_R32(tp, TxConfig);
5380	if (txconfig == ~0U)
5381	return dev_err_probe(dev: &pdev->dev, err: -EIO, fmt: "PCI read failed\n");
5382
5383	xid = (txconfig >> 20) & 0xfcf;
5384
5385	/* Identify chip attached to board */
5386	chipset = rtl8169_get_mac_version(xid, gmii: tp->supports_gmii);
5387	if (chipset == RTL_GIGA_MAC_NONE)
5388	return dev_err_probe(dev: &pdev->dev, err: -ENODEV,
5389	fmt: "unknown chip XID %03x, contact r8169 maintainers (see MAINTAINERS file)\n",
5390	xid);
5391	tp->mac_version = chipset;
5392
5393	/* Disable ASPM L1 as that cause random device stop working
5394	* problems as well as full system hangs for some PCIe devices users.
5395	*/
5396	if (rtl_aspm_is_safe(tp))
5397	rc = 0;
5398	else
5399	rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1);
5400	tp->aspm_manageable = !rc;
5401
5402	tp->dash_type = rtl_get_dash_type(tp);
5403	tp->dash_enabled = rtl_dash_is_enabled(tp);
5404
5405	tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK;
5406
5407	if (sizeof(dma_addr_t) > 4 && tp->mac_version >= RTL_GIGA_MAC_VER_18 &&
5408	!dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)))
5409	dev->features |= NETIF_F_HIGHDMA;
5410
5411	rtl_init_rxcfg(tp);
5412
5413	rtl8169_irq_mask_and_ack(tp);
5414
5415	rtl_hw_initialize(tp);
5416
5417	rtl_hw_reset(tp);
5418
5419	rc = rtl_alloc_irq(tp);
5420	if (rc < 0)
5421	return dev_err_probe(dev: &pdev->dev, err: rc, fmt: "Can't allocate interrupt\n");
5422
5423	tp->irq = pci_irq_vector(dev: pdev, nr: 0);
5424
5425	INIT_WORK(&tp->wk.work, rtl_task);
5426
5427	rtl_init_mac_address(tp);
5428
5429	dev->ethtool_ops = &rtl8169_ethtool_ops;
5430
5431	netif_napi_add(dev, napi: &tp->napi, poll: rtl8169_poll);
5432
5433	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
5434	NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
5435	dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
5436	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
5437
5438	/*
5439	* Pretend we are using VLANs; This bypasses a nasty bug where
5440	* Interrupts stop flowing on high load on 8110SCd controllers.
5441	*/
5442	if (tp->mac_version == RTL_GIGA_MAC_VER_05)
5443	/* Disallow toggling */
5444	dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
5445
5446	if (rtl_chip_supports_csum_v2(tp))
5447	dev->hw_features |= NETIF_F_IPV6_CSUM;
5448
5449	dev->features |= dev->hw_features;
5450
5451	/* There has been a number of reports that using SG/TSO results in
5452	* tx timeouts. However for a lot of people SG/TSO works fine.
5453	* Therefore disable both features by default, but allow users to
5454	* enable them. Use at own risk!
5455	*/
5456	if (rtl_chip_supports_csum_v2(tp)) {
5457	dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6;
5458	netif_set_tso_max_size(dev, RTL_GSO_MAX_SIZE_V2);
5459	netif_set_tso_max_segs(dev, RTL_GSO_MAX_SEGS_V2);
5460	} else {
5461	dev->hw_features |= NETIF_F_SG | NETIF_F_TSO;
5462	netif_set_tso_max_size(dev, RTL_GSO_MAX_SIZE_V1);
5463	netif_set_tso_max_segs(dev, RTL_GSO_MAX_SEGS_V1);
5464	}
5465
5466	dev->hw_features |= NETIF_F_RXALL;
5467	dev->hw_features |= NETIF_F_RXFCS;
5468
5469	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
5470
5471	netdev_sw_irq_coalesce_default_on(dev);
5472
5473	/* configure chip for default features */
5474	rtl8169_set_features(dev, features: dev->features);
5475
5476	if (!tp->dash_enabled) {
5477	rtl_set_d3_pll_down(tp, enable: true);
5478	} else {
5479	rtl_set_d3_pll_down(tp, enable: false);
5480	dev->wol_enabled = 1;
5481	}
5482
5483	jumbo_max = rtl_jumbo_max(tp);
5484	if (jumbo_max)
5485	dev->max_mtu = jumbo_max;
5486
5487	rtl_set_irq_mask(tp);
5488
5489	tp->fw_name = rtl_chip_infos[chipset].fw_name;
5490
5491	tp->counters = dmam_alloc_coherent (dev: &pdev->dev, size: sizeof(*tp->counters),
5492	dma_handle: &tp->counters_phys_addr,
5493	GFP_KERNEL);
5494	if (!tp->counters)
5495	return -ENOMEM;
5496
5497	pci_set_drvdata(pdev, data: tp);
5498
5499	rc = r8169_mdio_register(tp);
5500	if (rc)
5501	return rc;
5502
5503	rc = register_netdev(dev);
5504	if (rc)
5505	return rc;
5506
5507	if (IS_ENABLED(CONFIG_R8169_LEDS)) {
5508	if (rtl_is_8125(tp))
5509	tp->leds = rtl8125_init_leds(ndev: dev);
5510	else if (tp->mac_version > RTL_GIGA_MAC_VER_06)
5511	tp->leds = rtl8168_init_leds(ndev: dev);
5512	}
5513
5514	netdev_info(dev, format: "%s, %pM, XID %03x, IRQ %d\n",
5515	rtl_chip_infos[chipset].name, dev->dev_addr, xid, tp->irq);
5516
5517	if (jumbo_max)
5518	netdev_info(dev, format: "jumbo features [frames: %d bytes, tx checksumming: %s]\n",
5519	jumbo_max, tp->mac_version <= RTL_GIGA_MAC_VER_06 ?
5520	"ok" : "ko");
5521
5522	if (tp->dash_type != RTL_DASH_NONE) {
5523	netdev_info(dev, format: "DASH %s\n",
5524	tp->dash_enabled ? "enabled" : "disabled");
5525	rtl8168_driver_start(tp);
5526	}
5527
5528	if (pci_dev_run_wake(dev: pdev))
5529	pm_runtime_put_sync(dev: &pdev->dev);
5530
5531	return 0;
5532	}
5533
5534	static struct pci_driver rtl8169_pci_driver = {
5535	.name = KBUILD_MODNAME,
5536	.id_table = rtl8169_pci_tbl,
5537	.probe = rtl_init_one,
5538	.remove = rtl_remove_one,
5539	.shutdown = rtl_shutdown,
5540	.driver.pm = pm_ptr(&rtl8169_pm_ops),
5541	};
5542
5543	module_pci_driver(rtl8169_pci_driver);
5544