1	/* SPDX-License-Identifier: GPL-2.0+ */
2	/* Copyright (C) 2018 Microchip Technology Inc. */
3
4	#include <linux/module.h>
5	#include <linux/pci.h>
6	#include <linux/netdevice.h>
7	#include <linux/etherdevice.h>
8	#include <linux/crc32.h>
9	#include <linux/microchipphy.h>
10	#include <linux/net_tstamp.h>
11	#include <linux/of_mdio.h>
12	#include <linux/of_net.h>
13	#include <linux/phy.h>
14	#include <linux/phy_fixed.h>
15	#include <linux/rtnetlink.h>
16	#include <linux/iopoll.h>
17	#include <linux/crc16.h>
18	#include "lan743x_main.h"
19	#include "lan743x_ethtool.h"
20
21	#define MMD_ACCESS_ADDRESS 0
22	#define MMD_ACCESS_WRITE 1
23	#define MMD_ACCESS_READ 2
24	#define MMD_ACCESS_READ_INC 3
25	#define PCS_POWER_STATE_DOWN 0x6
26	#define PCS_POWER_STATE_UP 0x4
27
28	#define RFE_RD_FIFO_TH_3_DWORDS 0x3
29
30	static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter)
31	{
32	u32 chip_rev;
33	u32 cfg_load;
34	u32 hw_cfg;
35	u32 strap;
36	int ret;
37
38	/* Timeout = 100 (i.e. 1 sec (10 msce * 100)) */
39	ret = lan743x_hs_syslock_acquire(adapter, timeout: 100);
40	if (ret < 0) {
41	netif_err(adapter, drv, adapter->netdev,
42	"Sys Lock acquire failed ret:%d\n", ret);
43	return;
44	}
45
46	cfg_load = lan743x_csr_read(adapter, ETH_SYS_CONFIG_LOAD_STARTED_REG);
47	lan743x_hs_syslock_release(adapter);
48	hw_cfg = lan743x_csr_read(adapter, HW_CFG);
49
50	if (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ ||
51	hw_cfg & HW_CFG_RST_PROTECT_) {
52	strap = lan743x_csr_read(adapter, STRAP_READ);
53	if (strap & STRAP_READ_SGMII_EN_)
54	adapter->is_sgmii_en = true;
55	else
56	adapter->is_sgmii_en = false;
57	} else {
58	chip_rev = lan743x_csr_read(adapter, FPGA_REV);
59	if (chip_rev) {
60	if (chip_rev & FPGA_SGMII_OP)
61	adapter->is_sgmii_en = true;
62	else
63	adapter->is_sgmii_en = false;
64	} else {
65	adapter->is_sgmii_en = false;
66	}
67	}
68	netif_dbg(adapter, drv, adapter->netdev,
69	"SGMII I/F %sable\n", adapter->is_sgmii_en ? "En" : "Dis");
70	}
71
72	static bool is_pci11x1x_chip(struct lan743x_adapter *adapter)
73	{
74	struct lan743x_csr *csr = &adapter->csr;
75	u32 id_rev = csr->id_rev;
76
77	if (((id_rev & 0xFFFF0000) == ID_REV_ID_A011_) ||
78	((id_rev & 0xFFFF0000) == ID_REV_ID_A041_)) {
79	return true;
80	}
81	return false;
82	}
83
84	static void lan743x_pci_cleanup(struct lan743x_adapter *adapter)
85	{
86	pci_release_selected_regions(adapter->pdev,
87	pci_select_bars(dev: adapter->pdev,
88	IORESOURCE_MEM));
89	pci_disable_device(dev: adapter->pdev);
90	}
91
92	static int lan743x_pci_init(struct lan743x_adapter *adapter,
93	struct pci_dev *pdev)
94	{
95	unsigned long bars = 0;
96	int ret;
97
98	adapter->pdev = pdev;
99	ret = pci_enable_device_mem(dev: pdev);
100	if (ret)
101	goto return_error;
102
103	netif_info(adapter, probe, adapter->netdev,
104	"PCI: Vendor ID = 0x%04X, Device ID = 0x%04X\n",
105	pdev->vendor, pdev->device);
106	bars = pci_select_bars(dev: pdev, IORESOURCE_MEM);
107	if (!test_bit(0, &bars))
108	goto disable_device;
109
110	ret = pci_request_selected_regions(pdev, bars, DRIVER_NAME);
111	if (ret)
112	goto disable_device;
113
114	pci_set_master(dev: pdev);
115	return 0;
116
117	disable_device:
118	pci_disable_device(dev: adapter->pdev);
119
120	return_error:
121	return ret;
122	}
123
124	u32 lan743x_csr_read(struct lan743x_adapter *adapter, int offset)
125	{
126	return ioread32(&adapter->csr.csr_address[offset]);
127	}
128
129	void lan743x_csr_write(struct lan743x_adapter *adapter, int offset,
130	u32 data)
131	{
132	iowrite32(data, &adapter->csr.csr_address[offset]);
133	}
134
135	#define LAN743X_CSR_READ_OP(offset) lan743x_csr_read(adapter, offset)
136
137	static int lan743x_csr_light_reset(struct lan743x_adapter *adapter)
138	{
139	u32 data;
140
141	data = lan743x_csr_read(adapter, HW_CFG);
142	data |= HW_CFG_LRST_;
143	lan743x_csr_write(adapter, HW_CFG, data);
144
145	return readx_poll_timeout(LAN743X_CSR_READ_OP, HW_CFG, data,
146	!(data & HW_CFG_LRST_), 100000, 10000000);
147	}
148
149	static int lan743x_csr_wait_for_bit_atomic(struct lan743x_adapter *adapter,
150	int offset, u32 bit_mask,
151	int target_value, int udelay_min,
152	int udelay_max, int count)
153	{
154	u32 data;
155
156	return readx_poll_timeout_atomic(LAN743X_CSR_READ_OP, offset, data,
157	target_value == !!(data & bit_mask),
158	udelay_max, udelay_min * count);
159	}
160
161	static int lan743x_csr_wait_for_bit(struct lan743x_adapter *adapter,
162	int offset, u32 bit_mask,
163	int target_value, int usleep_min,
164	int usleep_max, int count)
165	{
166	u32 data;
167
168	return readx_poll_timeout(LAN743X_CSR_READ_OP, offset, data,
169	target_value == !!(data & bit_mask),
170	usleep_max, usleep_min * count);
171	}
172
173	static int lan743x_csr_init(struct lan743x_adapter *adapter)
174	{
175	struct lan743x_csr *csr = &adapter->csr;
176	resource_size_t bar_start, bar_length;
177
178	bar_start = pci_resource_start(adapter->pdev, 0);
179	bar_length = pci_resource_len(adapter->pdev, 0);
180	csr->csr_address = devm_ioremap(dev: &adapter->pdev->dev,
181	offset: bar_start, size: bar_length);
182	if (!csr->csr_address)
183	return -ENOMEM;
184
185	csr->id_rev = lan743x_csr_read(adapter, ID_REV);
186	csr->fpga_rev = lan743x_csr_read(adapter, FPGA_REV);
187	netif_info(adapter, probe, adapter->netdev,
188	"ID_REV = 0x%08X, FPGA_REV = %d.%d\n",
189	csr->id_rev, FPGA_REV_GET_MAJOR_(csr->fpga_rev),
190	FPGA_REV_GET_MINOR_(csr->fpga_rev));
191	if (!ID_REV_IS_VALID_CHIP_ID_(csr->id_rev))
192	return -ENODEV;
193
194	csr->flags = LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR;
195	switch (csr->id_rev & ID_REV_CHIP_REV_MASK_) {
196	case ID_REV_CHIP_REV_A0_:
197	csr->flags |= LAN743X_CSR_FLAG_IS_A0;
198	csr->flags &= ~LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR;
199	break;
200	case ID_REV_CHIP_REV_B0_:
201	csr->flags |= LAN743X_CSR_FLAG_IS_B0;
202	break;
203	}
204
205	return lan743x_csr_light_reset(adapter);
206	}
207
208	static void lan743x_intr_software_isr(struct lan743x_adapter *adapter)
209	{
210	struct lan743x_intr *intr = &adapter->intr;
211
212	/* disable the interrupt to prevent repeated re-triggering */
213	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_SW_GP_);
214	intr->software_isr_flag = true;
215	wake_up(&intr->software_isr_wq);
216	}
217
218	static void lan743x_tx_isr(void *context, u32 int_sts, u32 flags)
219	{
220	struct lan743x_tx *tx = context;
221	struct lan743x_adapter *adapter = tx->adapter;
222	bool enable_flag = true;
223
224	lan743x_csr_read(adapter, INT_EN_SET);
225	if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR) {
226	lan743x_csr_write(adapter, INT_EN_CLR,
227	INT_BIT_DMA_TX_(tx->channel_number));
228	}
229
230	if (int_sts & INT_BIT_DMA_TX_(tx->channel_number)) {
231	u32 ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
232	u32 dmac_int_sts;
233	u32 dmac_int_en;
234
235	if (flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ)
236	dmac_int_sts = lan743x_csr_read(adapter, DMAC_INT_STS);
237	else
238	dmac_int_sts = ioc_bit;
239	if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK)
240	dmac_int_en = lan743x_csr_read(adapter,
241	DMAC_INT_EN_SET);
242	else
243	dmac_int_en = ioc_bit;
244
245	dmac_int_en &= ioc_bit;
246	dmac_int_sts &= dmac_int_en;
247	if (dmac_int_sts & ioc_bit) {
248	napi_schedule(n: &tx->napi);
249	enable_flag = false;/* poll func will enable later */
250	}
251	}
252
253	if (enable_flag)
254	/* enable isr */
255	lan743x_csr_write(adapter, INT_EN_SET,
256	INT_BIT_DMA_TX_(tx->channel_number));
257	}
258
259	static void lan743x_rx_isr(void *context, u32 int_sts, u32 flags)
260	{
261	struct lan743x_rx *rx = context;
262	struct lan743x_adapter *adapter = rx->adapter;
263	bool enable_flag = true;
264
265	if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR) {
266	lan743x_csr_write(adapter, INT_EN_CLR,
267	INT_BIT_DMA_RX_(rx->channel_number));
268	}
269
270	if (int_sts & INT_BIT_DMA_RX_(rx->channel_number)) {
271	u32 rx_frame_bit = DMAC_INT_BIT_RXFRM_(rx->channel_number);
272	u32 dmac_int_sts;
273	u32 dmac_int_en;
274
275	if (flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ)
276	dmac_int_sts = lan743x_csr_read(adapter, DMAC_INT_STS);
277	else
278	dmac_int_sts = rx_frame_bit;
279	if (flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK)
280	dmac_int_en = lan743x_csr_read(adapter,
281	DMAC_INT_EN_SET);
282	else
283	dmac_int_en = rx_frame_bit;
284
285	dmac_int_en &= rx_frame_bit;
286	dmac_int_sts &= dmac_int_en;
287	if (dmac_int_sts & rx_frame_bit) {
288	napi_schedule(n: &rx->napi);
289	enable_flag = false;/* poll funct will enable later */
290	}
291	}
292
293	if (enable_flag) {
294	/* enable isr */
295	lan743x_csr_write(adapter, INT_EN_SET,
296	INT_BIT_DMA_RX_(rx->channel_number));
297	}
298	}
299
300	static void lan743x_intr_shared_isr(void *context, u32 int_sts, u32 flags)
301	{
302	struct lan743x_adapter *adapter = context;
303	unsigned int channel;
304
305	if (int_sts & INT_BIT_ALL_RX_) {
306	for (channel = 0; channel < LAN743X_USED_RX_CHANNELS;
307	channel++) {
308	u32 int_bit = INT_BIT_DMA_RX_(channel);
309
310	if (int_sts & int_bit) {
311	lan743x_rx_isr(context: &adapter->rx[channel],
312	int_sts: int_bit, flags);
313	int_sts &= ~int_bit;
314	}
315	}
316	}
317	if (int_sts & INT_BIT_ALL_TX_) {
318	for (channel = 0; channel < adapter->used_tx_channels;
319	channel++) {
320	u32 int_bit = INT_BIT_DMA_TX_(channel);
321
322	if (int_sts & int_bit) {
323	lan743x_tx_isr(context: &adapter->tx[channel],
324	int_sts: int_bit, flags);
325	int_sts &= ~int_bit;
326	}
327	}
328	}
329	if (int_sts & INT_BIT_ALL_OTHER_) {
330	if (int_sts & INT_BIT_SW_GP_) {
331	lan743x_intr_software_isr(adapter);
332	int_sts &= ~INT_BIT_SW_GP_;
333	}
334	if (int_sts & INT_BIT_1588_) {
335	lan743x_ptp_isr(context: adapter);
336	int_sts &= ~INT_BIT_1588_;
337	}
338	}
339	if (int_sts)
340	lan743x_csr_write(adapter, INT_EN_CLR, data: int_sts);
341	}
342
343	static irqreturn_t lan743x_intr_entry_isr(int irq, void *ptr)
344	{
345	struct lan743x_vector *vector = ptr;
346	struct lan743x_adapter *adapter = vector->adapter;
347	irqreturn_t result = IRQ_NONE;
348	u32 int_enables;
349	u32 int_sts;
350
351	if (vector->flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ) {
352	int_sts = lan743x_csr_read(adapter, INT_STS);
353	} else if (vector->flags &
354	(LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C |
355	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)) {
356	int_sts = lan743x_csr_read(adapter, INT_STS_R2C);
357	} else {
358	/* use mask as implied status */
359	int_sts = vector->int_mask | INT_BIT_MAS_;
360	}
361
362	if (!(int_sts & INT_BIT_MAS_))
363	goto irq_done;
364
365	if (vector->flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR)
366	/* disable vector interrupt */
367	lan743x_csr_write(adapter,
368	INT_VEC_EN_CLR,
369	INT_VEC_EN_(vector->vector_index));
370
371	if (vector->flags & LAN743X_VECTOR_FLAG_MASTER_ENABLE_CLEAR)
372	/* disable master interrupt */
373	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_MAS_);
374
375	if (vector->flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK) {
376	int_enables = lan743x_csr_read(adapter, INT_EN_SET);
377	} else {
378	/* use vector mask as implied enable mask */
379	int_enables = vector->int_mask;
380	}
381
382	int_sts &= int_enables;
383	int_sts &= vector->int_mask;
384	if (int_sts) {
385	if (vector->handler) {
386	vector->handler(vector->context,
387	int_sts, vector->flags);
388	} else {
389	/* disable interrupts on this vector */
390	lan743x_csr_write(adapter, INT_EN_CLR,
391	data: vector->int_mask);
392	}
393	result = IRQ_HANDLED;
394	}
395
396	if (vector->flags & LAN743X_VECTOR_FLAG_MASTER_ENABLE_SET)
397	/* enable master interrupt */
398	lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_MAS_);
399
400	if (vector->flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET)
401	/* enable vector interrupt */
402	lan743x_csr_write(adapter,
403	INT_VEC_EN_SET,
404	INT_VEC_EN_(vector->vector_index));
405	irq_done:
406	return result;
407	}
408
409	static int lan743x_intr_test_isr(struct lan743x_adapter *adapter)
410	{
411	struct lan743x_intr *intr = &adapter->intr;
412	int ret;
413
414	intr->software_isr_flag = false;
415
416	/* enable and activate test interrupt */
417	lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_SW_GP_);
418	lan743x_csr_write(adapter, INT_SET, INT_BIT_SW_GP_);
419
420	ret = wait_event_timeout(intr->software_isr_wq,
421	intr->software_isr_flag,
422	msecs_to_jiffies(200));
423
424	/* disable test interrupt */
425	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_SW_GP_);
426
427	return ret > 0 ? 0 : -ENODEV;
428	}
429
430	static int lan743x_intr_register_isr(struct lan743x_adapter *adapter,
431	int vector_index, u32 flags,
432	u32 int_mask,
433	lan743x_vector_handler handler,
434	void *context)
435	{
436	struct lan743x_vector *vector = &adapter->intr.vector_list
437	[vector_index];
438	int ret;
439
440	vector->adapter = adapter;
441	vector->flags = flags;
442	vector->vector_index = vector_index;
443	vector->int_mask = int_mask;
444	vector->handler = handler;
445	vector->context = context;
446
447	ret = request_irq(irq: vector->irq,
448	handler: lan743x_intr_entry_isr,
449	flags: (flags & LAN743X_VECTOR_FLAG_IRQ_SHARED) ?
450	IRQF_SHARED : 0, DRIVER_NAME, dev: vector);
451	if (ret) {
452	vector->handler = NULL;
453	vector->context = NULL;
454	vector->int_mask = 0;
455	vector->flags = 0;
456	}
457	return ret;
458	}
459
460	static void lan743x_intr_unregister_isr(struct lan743x_adapter *adapter,
461	int vector_index)
462	{
463	struct lan743x_vector *vector = &adapter->intr.vector_list
464	[vector_index];
465
466	free_irq(vector->irq, vector);
467	vector->handler = NULL;
468	vector->context = NULL;
469	vector->int_mask = 0;
470	vector->flags = 0;
471	}
472
473	static u32 lan743x_intr_get_vector_flags(struct lan743x_adapter *adapter,
474	u32 int_mask)
475	{
476	int index;
477
478	for (index = 0; index < adapter->max_vector_count; index++) {
479	if (adapter->intr.vector_list[index].int_mask & int_mask)
480	return adapter->intr.vector_list[index].flags;
481	}
482	return 0;
483	}
484
485	static void lan743x_intr_close(struct lan743x_adapter *adapter)
486	{
487	struct lan743x_intr *intr = &adapter->intr;
488	int index = 0;
489
490	lan743x_csr_write(adapter, INT_EN_CLR, INT_BIT_MAS_);
491	if (adapter->is_pci11x1x)
492	lan743x_csr_write(adapter, INT_VEC_EN_CLR, data: 0x0000FFFF);
493	else
494	lan743x_csr_write(adapter, INT_VEC_EN_CLR, data: 0x000000FF);
495
496	for (index = 0; index < intr->number_of_vectors; index++) {
497	if (intr->flags & INTR_FLAG_IRQ_REQUESTED(index)) {
498	lan743x_intr_unregister_isr(adapter, vector_index: index);
499	intr->flags &= ~INTR_FLAG_IRQ_REQUESTED(index);
500	}
501	}
502
503	if (intr->flags & INTR_FLAG_MSI_ENABLED) {
504	pci_disable_msi(dev: adapter->pdev);
505	intr->flags &= ~INTR_FLAG_MSI_ENABLED;
506	}
507
508	if (intr->flags & INTR_FLAG_MSIX_ENABLED) {
509	pci_disable_msix(dev: adapter->pdev);
510	intr->flags &= ~INTR_FLAG_MSIX_ENABLED;
511	}
512	}
513
514	static int lan743x_intr_open(struct lan743x_adapter *adapter)
515	{
516	struct msix_entry msix_entries[PCI11X1X_MAX_VECTOR_COUNT];
517	struct lan743x_intr *intr = &adapter->intr;
518	unsigned int used_tx_channels;
519	u32 int_vec_en_auto_clr = 0;
520	u8 max_vector_count;
521	u32 int_vec_map0 = 0;
522	u32 int_vec_map1 = 0;
523	int ret = -ENODEV;
524	int index = 0;
525	u32 flags = 0;
526
527	intr->number_of_vectors = 0;
528
529	/* Try to set up MSIX interrupts */
530	max_vector_count = adapter->max_vector_count;
531	memset(&msix_entries[0], 0,
532	sizeof(struct msix_entry) * max_vector_count);
533	for (index = 0; index < max_vector_count; index++)
534	msix_entries[index].entry = index;
535	used_tx_channels = adapter->used_tx_channels;
536	ret = pci_enable_msix_range(dev: adapter->pdev,
537	entries: msix_entries, minvec: 1,
538	maxvec: 1 + used_tx_channels +
539	LAN743X_USED_RX_CHANNELS);
540
541	if (ret > 0) {
542	intr->flags |= INTR_FLAG_MSIX_ENABLED;
543	intr->number_of_vectors = ret;
544	intr->using_vectors = true;
545	for (index = 0; index < intr->number_of_vectors; index++)
546	intr->vector_list[index].irq = msix_entries
547	[index].vector;
548	netif_info(adapter, ifup, adapter->netdev,
549	"using MSIX interrupts, number of vectors = %d\n",
550	intr->number_of_vectors);
551	}
552
553	/* If MSIX failed try to setup using MSI interrupts */
554	if (!intr->number_of_vectors) {
555	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
556	if (!pci_enable_msi(dev: adapter->pdev)) {
557	intr->flags |= INTR_FLAG_MSI_ENABLED;
558	intr->number_of_vectors = 1;
559	intr->using_vectors = true;
560	intr->vector_list[0].irq =
561	adapter->pdev->irq;
562	netif_info(adapter, ifup, adapter->netdev,
563	"using MSI interrupts, number of vectors = %d\n",
564	intr->number_of_vectors);
565	}
566	}
567	}
568
569	/* If MSIX, and MSI failed, setup using legacy interrupt */
570	if (!intr->number_of_vectors) {
571	intr->number_of_vectors = 1;
572	intr->using_vectors = false;
573	intr->vector_list[0].irq = intr->irq;
574	netif_info(adapter, ifup, adapter->netdev,
575	"using legacy interrupts\n");
576	}
577
578	/* At this point we must have at least one irq */
579	lan743x_csr_write(adapter, INT_VEC_EN_CLR, data: 0xFFFFFFFF);
580
581	/* map all interrupts to vector 0 */
582	lan743x_csr_write(adapter, INT_VEC_MAP0, data: 0x00000000);
583	lan743x_csr_write(adapter, INT_VEC_MAP1, data: 0x00000000);
584	lan743x_csr_write(adapter, INT_VEC_MAP2, data: 0x00000000);
585	flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
586	LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
587	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
588	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR;
589
590	if (intr->using_vectors) {
591	flags |= LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
592	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
593	} else {
594	flags |= LAN743X_VECTOR_FLAG_MASTER_ENABLE_CLEAR |
595	LAN743X_VECTOR_FLAG_MASTER_ENABLE_SET |
596	LAN743X_VECTOR_FLAG_IRQ_SHARED;
597	}
598
599	if (adapter->csr.flags & LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
600	flags &= ~LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ;
601	flags &= ~LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C;
602	flags &= ~LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR;
603	flags &= ~LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK;
604	flags |= LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C;
605	flags |= LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C;
606	}
607
608	init_waitqueue_head(&intr->software_isr_wq);
609
610	ret = lan743x_intr_register_isr(adapter, vector_index: 0, flags,
611	INT_BIT_ALL_RX_ | INT_BIT_ALL_TX_ |
612	INT_BIT_ALL_OTHER_,
613	handler: lan743x_intr_shared_isr, context: adapter);
614	if (ret)
615	goto clean_up;
616	intr->flags |= INTR_FLAG_IRQ_REQUESTED(0);
617
618	if (intr->using_vectors)
619	lan743x_csr_write(adapter, INT_VEC_EN_SET,
620	INT_VEC_EN_(0));
621
622	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
623	lan743x_csr_write(adapter, INT_MOD_CFG0, LAN743X_INT_MOD);
624	lan743x_csr_write(adapter, INT_MOD_CFG1, LAN743X_INT_MOD);
625	lan743x_csr_write(adapter, INT_MOD_CFG2, LAN743X_INT_MOD);
626	lan743x_csr_write(adapter, INT_MOD_CFG3, LAN743X_INT_MOD);
627	lan743x_csr_write(adapter, INT_MOD_CFG4, LAN743X_INT_MOD);
628	lan743x_csr_write(adapter, INT_MOD_CFG5, LAN743X_INT_MOD);
629	lan743x_csr_write(adapter, INT_MOD_CFG6, LAN743X_INT_MOD);
630	lan743x_csr_write(adapter, INT_MOD_CFG7, LAN743X_INT_MOD);
631	if (adapter->is_pci11x1x) {
632	lan743x_csr_write(adapter, INT_MOD_CFG8, LAN743X_INT_MOD);
633	lan743x_csr_write(adapter, INT_MOD_CFG9, LAN743X_INT_MOD);
634	lan743x_csr_write(adapter, INT_MOD_MAP0, data: 0x00007654);
635	lan743x_csr_write(adapter, INT_MOD_MAP1, data: 0x00003210);
636	} else {
637	lan743x_csr_write(adapter, INT_MOD_MAP0, data: 0x00005432);
638	lan743x_csr_write(adapter, INT_MOD_MAP1, data: 0x00000001);
639	}
640	lan743x_csr_write(adapter, INT_MOD_MAP2, data: 0x00FFFFFF);
641	}
642
643	/* enable interrupts */
644	lan743x_csr_write(adapter, INT_EN_SET, INT_BIT_MAS_);
645	ret = lan743x_intr_test_isr(adapter);
646	if (ret)
647	goto clean_up;
648
649	if (intr->number_of_vectors > 1) {
650	int number_of_tx_vectors = intr->number_of_vectors - 1;
651
652	if (number_of_tx_vectors > used_tx_channels)
653	number_of_tx_vectors = used_tx_channels;
654	flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
655	LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
656	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
657	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR |
658	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
659	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
660
661	if (adapter->csr.flags &
662	LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
663	flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET |
664	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET |
665	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR |
666	LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR;
667	}
668
669	for (index = 0; index < number_of_tx_vectors; index++) {
670	u32 int_bit = INT_BIT_DMA_TX_(index);
671	int vector = index + 1;
672
673	/* map TX interrupt to vector */
674	int_vec_map1 |= INT_VEC_MAP1_TX_VEC_(index, vector);
675	lan743x_csr_write(adapter, INT_VEC_MAP1, data: int_vec_map1);
676
677	/* Remove TX interrupt from shared mask */
678	intr->vector_list[0].int_mask &= ~int_bit;
679	ret = lan743x_intr_register_isr(adapter, vector_index: vector, flags,
680	int_mask: int_bit, handler: lan743x_tx_isr,
681	context: &adapter->tx[index]);
682	if (ret)
683	goto clean_up;
684	intr->flags |= INTR_FLAG_IRQ_REQUESTED(vector);
685	if (!(flags &
686	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET))
687	lan743x_csr_write(adapter, INT_VEC_EN_SET,
688	INT_VEC_EN_(vector));
689	}
690	}
691	if ((intr->number_of_vectors - used_tx_channels) > 1) {
692	int number_of_rx_vectors = intr->number_of_vectors -
693	used_tx_channels - 1;
694
695	if (number_of_rx_vectors > LAN743X_USED_RX_CHANNELS)
696	number_of_rx_vectors = LAN743X_USED_RX_CHANNELS;
697
698	flags = LAN743X_VECTOR_FLAG_SOURCE_STATUS_READ |
699	LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C |
700	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CHECK |
701	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_CLEAR |
702	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_CLEAR |
703	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_ISR_SET;
704
705	if (adapter->csr.flags &
706	LAN743X_CSR_FLAG_SUPPORTS_INTR_AUTO_SET_CLR) {
707	flags = LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR |
708	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET |
709	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET |
710	LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR |
711	LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR;
712	}
713	for (index = 0; index < number_of_rx_vectors; index++) {
714	int vector = index + 1 + used_tx_channels;
715	u32 int_bit = INT_BIT_DMA_RX_(index);
716
717	/* map RX interrupt to vector */
718	int_vec_map0 |= INT_VEC_MAP0_RX_VEC_(index, vector);
719	lan743x_csr_write(adapter, INT_VEC_MAP0, data: int_vec_map0);
720	if (flags &
721	LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_CLEAR) {
722	int_vec_en_auto_clr |= INT_VEC_EN_(vector);
723	lan743x_csr_write(adapter, INT_VEC_EN_AUTO_CLR,
724	data: int_vec_en_auto_clr);
725	}
726
727	/* Remove RX interrupt from shared mask */
728	intr->vector_list[0].int_mask &= ~int_bit;
729	ret = lan743x_intr_register_isr(adapter, vector_index: vector, flags,
730	int_mask: int_bit, handler: lan743x_rx_isr,
731	context: &adapter->rx[index]);
732	if (ret)
733	goto clean_up;
734	intr->flags |= INTR_FLAG_IRQ_REQUESTED(vector);
735
736	lan743x_csr_write(adapter, INT_VEC_EN_SET,
737	INT_VEC_EN_(vector));
738	}
739	}
740	return 0;
741
742	clean_up:
743	lan743x_intr_close(adapter);
744	return ret;
745	}
746
747	static int lan743x_dp_write(struct lan743x_adapter *adapter,
748	u32 select, u32 addr, u32 length, u32 *buf)
749	{
750	u32 dp_sel;
751	int i;
752
753	if (lan743x_csr_wait_for_bit_atomic(adapter, DP_SEL, DP_SEL_DPRDY_,
754	target_value: 1, udelay_min: 40, udelay_max: 100, count: 100))
755	return -EIO;
756	dp_sel = lan743x_csr_read(adapter, DP_SEL);
757	dp_sel &= ~DP_SEL_MASK_;
758	dp_sel |= select;
759	lan743x_csr_write(adapter, DP_SEL, data: dp_sel);
760
761	for (i = 0; i < length; i++) {
762	lan743x_csr_write(adapter, DP_ADDR, data: addr + i);
763	lan743x_csr_write(adapter, DP_DATA_0, data: buf[i]);
764	lan743x_csr_write(adapter, DP_CMD, DP_CMD_WRITE_);
765	if (lan743x_csr_wait_for_bit_atomic(adapter, DP_SEL,
766	DP_SEL_DPRDY_,
767	target_value: 1, udelay_min: 40, udelay_max: 100, count: 100))
768	return -EIO;
769	}
770
771	return 0;
772	}
773
774	static u32 lan743x_mac_mii_access(u16 id, u16 index, int read)
775	{
776	u32 ret;
777
778	ret = (id << MAC_MII_ACC_PHY_ADDR_SHIFT_) &
779	MAC_MII_ACC_PHY_ADDR_MASK_;
780	ret |= (index << MAC_MII_ACC_MIIRINDA_SHIFT_) &
781	MAC_MII_ACC_MIIRINDA_MASK_;
782
783	if (read)
784	ret |= MAC_MII_ACC_MII_READ_;
785	else
786	ret |= MAC_MII_ACC_MII_WRITE_;
787	ret |= MAC_MII_ACC_MII_BUSY_;
788
789	return ret;
790	}
791
792	static int lan743x_mac_mii_wait_till_not_busy(struct lan743x_adapter *adapter)
793	{
794	u32 data;
795
796	return readx_poll_timeout(LAN743X_CSR_READ_OP, MAC_MII_ACC, data,
797	!(data & MAC_MII_ACC_MII_BUSY_), 0, 1000000);
798	}
799
800	static int lan743x_mdiobus_read_c22(struct mii_bus *bus, int phy_id, int index)
801	{
802	struct lan743x_adapter *adapter = bus->priv;
803	u32 val, mii_access;
804	int ret;
805
806	/* comfirm MII not busy */
807	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
808	if (ret < 0)
809	return ret;
810
811	/* set the address, index & direction (read from PHY) */
812	mii_access = lan743x_mac_mii_access(id: phy_id, index, MAC_MII_READ);
813	lan743x_csr_write(adapter, MAC_MII_ACC, data: mii_access);
814	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
815	if (ret < 0)
816	return ret;
817
818	val = lan743x_csr_read(adapter, MAC_MII_DATA);
819	return (int)(val & 0xFFFF);
820	}
821
822	static int lan743x_mdiobus_write_c22(struct mii_bus *bus,
823	int phy_id, int index, u16 regval)
824	{
825	struct lan743x_adapter *adapter = bus->priv;
826	u32 val, mii_access;
827	int ret;
828
829	/* confirm MII not busy */
830	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
831	if (ret < 0)
832	return ret;
833	val = (u32)regval;
834	lan743x_csr_write(adapter, MAC_MII_DATA, data: val);
835
836	/* set the address, index & direction (write to PHY) */
837	mii_access = lan743x_mac_mii_access(id: phy_id, index, MAC_MII_WRITE);
838	lan743x_csr_write(adapter, MAC_MII_ACC, data: mii_access);
839	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
840	return ret;
841	}
842
843	static u32 lan743x_mac_mmd_access(int id, int dev_addr, int op)
844	{
845	u32 ret;
846
847	ret = (id << MAC_MII_ACC_PHY_ADDR_SHIFT_) &
848	MAC_MII_ACC_PHY_ADDR_MASK_;
849	ret |= (dev_addr << MAC_MII_ACC_MIIMMD_SHIFT_) &
850	MAC_MII_ACC_MIIMMD_MASK_;
851	if (op == MMD_ACCESS_WRITE)
852	ret |= MAC_MII_ACC_MIICMD_WRITE_;
853	else if (op == MMD_ACCESS_READ)
854	ret |= MAC_MII_ACC_MIICMD_READ_;
855	else if (op == MMD_ACCESS_READ_INC)
856	ret |= MAC_MII_ACC_MIICMD_READ_INC_;
857	else
858	ret |= MAC_MII_ACC_MIICMD_ADDR_;
859	ret |= (MAC_MII_ACC_MII_BUSY_ | MAC_MII_ACC_MIICL45_);
860
861	return ret;
862	}
863
864	static int lan743x_mdiobus_read_c45(struct mii_bus *bus, int phy_id,
865	int dev_addr, int index)
866	{
867	struct lan743x_adapter *adapter = bus->priv;
868	u32 mmd_access;
869	int ret;
870
871	/* comfirm MII not busy */
872	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
873	if (ret < 0)
874	return ret;
875
876	/* Load Register Address */
877	lan743x_csr_write(adapter, MAC_MII_DATA, data: index);
878	mmd_access = lan743x_mac_mmd_access(id: phy_id, dev_addr,
879	MMD_ACCESS_ADDRESS);
880	lan743x_csr_write(adapter, MAC_MII_ACC, data: mmd_access);
881	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
882	if (ret < 0)
883	return ret;
884
885	/* Read Data */
886	mmd_access = lan743x_mac_mmd_access(id: phy_id, dev_addr,
887	MMD_ACCESS_READ);
888	lan743x_csr_write(adapter, MAC_MII_ACC, data: mmd_access);
889	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
890	if (ret < 0)
891	return ret;
892
893	ret = lan743x_csr_read(adapter, MAC_MII_DATA);
894	return (int)(ret & 0xFFFF);
895	}
896
897	static int lan743x_mdiobus_write_c45(struct mii_bus *bus, int phy_id,
898	int dev_addr, int index, u16 regval)
899	{
900	struct lan743x_adapter *adapter = bus->priv;
901	u32 mmd_access;
902	int ret;
903
904	/* confirm MII not busy */
905	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
906	if (ret < 0)
907	return ret;
908
909	/* Load Register Address */
910	lan743x_csr_write(adapter, MAC_MII_DATA, data: (u32)index);
911	mmd_access = lan743x_mac_mmd_access(id: phy_id, dev_addr,
912	MMD_ACCESS_ADDRESS);
913	lan743x_csr_write(adapter, MAC_MII_ACC, data: mmd_access);
914	ret = lan743x_mac_mii_wait_till_not_busy(adapter);
915	if (ret < 0)
916	return ret;
917
918	/* Write Data */
919	lan743x_csr_write(adapter, MAC_MII_DATA, data: (u32)regval);
920	mmd_access = lan743x_mac_mmd_access(id: phy_id, dev_addr,
921	MMD_ACCESS_WRITE);
922	lan743x_csr_write(adapter, MAC_MII_ACC, data: mmd_access);
923
924	return lan743x_mac_mii_wait_till_not_busy(adapter);
925	}
926
927	static int lan743x_sgmii_wait_till_not_busy(struct lan743x_adapter *adapter)
928	{
929	u32 data;
930	int ret;
931
932	ret = readx_poll_timeout(LAN743X_CSR_READ_OP, SGMII_ACC, data,
933	!(data & SGMII_ACC_SGMII_BZY_), 100, 1000000);
934	if (ret < 0)
935	netif_err(adapter, drv, adapter->netdev,
936	"%s: error %d sgmii wait timeout\n", __func__, ret);
937
938	return ret;
939	}
940
941	int lan743x_sgmii_read(struct lan743x_adapter *adapter, u8 mmd, u16 addr)
942	{
943	u32 mmd_access;
944	int ret;
945	u32 val;
946
947	if (mmd > 31) {
948	netif_err(adapter, probe, adapter->netdev,
949	"%s mmd should <= 31\n", __func__);
950	return -EINVAL;
951	}
952
953	mutex_lock(&adapter->sgmii_rw_lock);
954	/* Load Register Address */
955	mmd_access = mmd << SGMII_ACC_SGMII_MMD_SHIFT_;
956	mmd_access |= (addr | SGMII_ACC_SGMII_BZY_);
957	lan743x_csr_write(adapter, SGMII_ACC, data: mmd_access);
958	ret = lan743x_sgmii_wait_till_not_busy(adapter);
959	if (ret < 0)
960	goto sgmii_unlock;
961
962	val = lan743x_csr_read(adapter, SGMII_DATA);
963	ret = (int)(val & SGMII_DATA_MASK_);
964
965	sgmii_unlock:
966	mutex_unlock(lock: &adapter->sgmii_rw_lock);
967
968	return ret;
969	}
970
971	static int lan743x_sgmii_write(struct lan743x_adapter *adapter,
972	u8 mmd, u16 addr, u16 val)
973	{
974	u32 mmd_access;
975	int ret;
976
977	if (mmd > 31) {
978	netif_err(adapter, probe, adapter->netdev,
979	"%s mmd should <= 31\n", __func__);
980	return -EINVAL;
981	}
982	mutex_lock(&adapter->sgmii_rw_lock);
983	/* Load Register Data */
984	lan743x_csr_write(adapter, SGMII_DATA, data: (u32)(val & SGMII_DATA_MASK_));
985	/* Load Register Address */
986	mmd_access = mmd << SGMII_ACC_SGMII_MMD_SHIFT_;
987	mmd_access |= (addr | SGMII_ACC_SGMII_BZY_ | SGMII_ACC_SGMII_WR_);
988	lan743x_csr_write(adapter, SGMII_ACC, data: mmd_access);
989	ret = lan743x_sgmii_wait_till_not_busy(adapter);
990	mutex_unlock(lock: &adapter->sgmii_rw_lock);
991
992	return ret;
993	}
994
995	static int lan743x_sgmii_mpll_set(struct lan743x_adapter *adapter,
996	u16 baud)
997	{
998	int mpllctrl0;
999	int mpllctrl1;
1000	int miscctrl1;
1001	int ret;
1002
1003	mpllctrl0 = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1004	VR_MII_GEN2_4_MPLL_CTRL0);
1005	if (mpllctrl0 < 0)
1006	return mpllctrl0;
1007
1008	mpllctrl0 &= ~VR_MII_MPLL_CTRL0_USE_REFCLK_PAD_;
1009	if (baud == VR_MII_BAUD_RATE_1P25GBPS) {
1010	mpllctrl1 = VR_MII_MPLL_MULTIPLIER_100;
1011	/* mpll_baud_clk/4 */
1012	miscctrl1 = 0xA;
1013	} else {
1014	mpllctrl1 = VR_MII_MPLL_MULTIPLIER_125;
1015	/* mpll_baud_clk/2 */
1016	miscctrl1 = 0x5;
1017	}
1018
1019	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1020	VR_MII_GEN2_4_MPLL_CTRL0, val: mpllctrl0);
1021	if (ret < 0)
1022	return ret;
1023
1024	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1025	VR_MII_GEN2_4_MPLL_CTRL1, val: mpllctrl1);
1026	if (ret < 0)
1027	return ret;
1028
1029	return lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1030	VR_MII_GEN2_4_MISC_CTRL1, val: miscctrl1);
1031	}
1032
1033	static int lan743x_sgmii_2_5G_mode_set(struct lan743x_adapter *adapter,
1034	bool enable)
1035	{
1036	if (enable)
1037	return lan743x_sgmii_mpll_set(adapter,
1038	VR_MII_BAUD_RATE_3P125GBPS);
1039	else
1040	return lan743x_sgmii_mpll_set(adapter,
1041	VR_MII_BAUD_RATE_1P25GBPS);
1042	}
1043
1044	static int lan743x_is_sgmii_2_5G_mode(struct lan743x_adapter *adapter,
1045	bool *status)
1046	{
1047	int ret;
1048
1049	ret = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1050	VR_MII_GEN2_4_MPLL_CTRL1);
1051	if (ret < 0)
1052	return ret;
1053
1054	if (ret == VR_MII_MPLL_MULTIPLIER_125 ||
1055	ret == VR_MII_MPLL_MULTIPLIER_50)
1056	*status = true;
1057	else
1058	*status = false;
1059
1060	return 0;
1061	}
1062
1063	static int lan743x_sgmii_aneg_update(struct lan743x_adapter *adapter)
1064	{
1065	enum lan743x_sgmii_lsd lsd = adapter->sgmii_lsd;
1066	int mii_ctrl;
1067	int dgt_ctrl;
1068	int an_ctrl;
1069	int ret;
1070
1071	if (lsd == LINK_2500_MASTER || lsd == LINK_2500_SLAVE)
1072	/* Switch to 2.5 Gbps */
1073	ret = lan743x_sgmii_2_5G_mode_set(adapter, enable: true);
1074	else
1075	/* Switch to 10/100/1000 Mbps clock */
1076	ret = lan743x_sgmii_2_5G_mode_set(adapter, enable: false);
1077	if (ret < 0)
1078	return ret;
1079
1080	/* Enable SGMII Auto NEG */
1081	mii_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, MII_BMCR);
1082	if (mii_ctrl < 0)
1083	return mii_ctrl;
1084
1085	an_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, VR_MII_AN_CTRL);
1086	if (an_ctrl < 0)
1087	return an_ctrl;
1088
1089	dgt_ctrl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1090	VR_MII_DIG_CTRL1);
1091	if (dgt_ctrl < 0)
1092	return dgt_ctrl;
1093
1094	if (lsd == LINK_2500_MASTER || lsd == LINK_2500_SLAVE) {
1095	mii_ctrl &= ~(BMCR_ANENABLE | BMCR_ANRESTART | BMCR_SPEED100);
1096	mii_ctrl |= BMCR_SPEED1000;
1097	dgt_ctrl |= VR_MII_DIG_CTRL1_CL37_TMR_OVR_RIDE_;
1098	dgt_ctrl &= ~VR_MII_DIG_CTRL1_MAC_AUTO_SW_;
1099	/* In order for Auto-Negotiation to operate properly at
1100	* 2.5 Gbps the 1.6ms link timer values must be adjusted
1101	* The VR_MII_LINK_TIMER_CTRL Register must be set to
1102	* 16'h7A1 and The CL37_TMR_OVR_RIDE bit of the
1103	* VR_MII_DIG_CTRL1 Register set to 1
1104	*/
1105	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1106	VR_MII_LINK_TIMER_CTRL, val: 0x7A1);
1107	if (ret < 0)
1108	return ret;
1109	} else {
1110	mii_ctrl |= (BMCR_ANENABLE | BMCR_ANRESTART);
1111	an_ctrl &= ~VR_MII_AN_CTRL_SGMII_LINK_STS_;
1112	dgt_ctrl &= ~VR_MII_DIG_CTRL1_CL37_TMR_OVR_RIDE_;
1113	dgt_ctrl |= VR_MII_DIG_CTRL1_MAC_AUTO_SW_;
1114	}
1115
1116	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR,
1117	val: mii_ctrl);
1118	if (ret < 0)
1119	return ret;
1120
1121	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1122	VR_MII_DIG_CTRL1, val: dgt_ctrl);
1123	if (ret < 0)
1124	return ret;
1125
1126	return lan743x_sgmii_write(adapter, MDIO_MMD_VEND2,
1127	VR_MII_AN_CTRL, val: an_ctrl);
1128	}
1129
1130	static int lan743x_pcs_seq_state(struct lan743x_adapter *adapter, u8 state)
1131	{
1132	u8 wait_cnt = 0;
1133	u32 dig_sts;
1134
1135	do {
1136	dig_sts = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2,
1137	VR_MII_DIG_STS);
1138	if (((dig_sts & VR_MII_DIG_STS_PSEQ_STATE_MASK_) >>
1139	VR_MII_DIG_STS_PSEQ_STATE_POS_) == state)
1140	break;
1141	usleep_range(min: 1000, max: 2000);
1142	} while (wait_cnt++ < 10);
1143
1144	if (wait_cnt >= 10)
1145	return -ETIMEDOUT;
1146
1147	return 0;
1148	}
1149
1150	static int lan743x_sgmii_config(struct lan743x_adapter *adapter)
1151	{
1152	struct net_device *netdev = adapter->netdev;
1153	struct phy_device *phydev = netdev->phydev;
1154	enum lan743x_sgmii_lsd lsd = POWER_DOWN;
1155	int mii_ctl;
1156	bool status;
1157	int ret;
1158
1159	switch (phydev->speed) {
1160	case SPEED_2500:
1161	if (phydev->master_slave_state == MASTER_SLAVE_STATE_MASTER)
1162	lsd = LINK_2500_MASTER;
1163	else
1164	lsd = LINK_2500_SLAVE;
1165	break;
1166	case SPEED_1000:
1167	if (phydev->master_slave_state == MASTER_SLAVE_STATE_MASTER)
1168	lsd = LINK_1000_MASTER;
1169	else
1170	lsd = LINK_1000_SLAVE;
1171	break;
1172	case SPEED_100:
1173	if (phydev->duplex)
1174	lsd = LINK_100FD;
1175	else
1176	lsd = LINK_100HD;
1177	break;
1178	case SPEED_10:
1179	if (phydev->duplex)
1180	lsd = LINK_10FD;
1181	else
1182	lsd = LINK_10HD;
1183	break;
1184	default:
1185	netif_err(adapter, drv, adapter->netdev,
1186	"Invalid speed %d\n", phydev->speed);
1187	return -EINVAL;
1188	}
1189
1190	adapter->sgmii_lsd = lsd;
1191	ret = lan743x_sgmii_aneg_update(adapter);
1192	if (ret < 0) {
1193	netif_err(adapter, drv, adapter->netdev,
1194	"error %d SGMII cfg failed\n", ret);
1195	return ret;
1196	}
1197
1198	ret = lan743x_is_sgmii_2_5G_mode(adapter, status: &status);
1199	if (ret < 0) {
1200	netif_err(adapter, drv, adapter->netdev,
1201	"error %d SGMII get mode failed\n", ret);
1202	return ret;
1203	}
1204
1205	if (status)
1206	netif_dbg(adapter, drv, adapter->netdev,
1207	"SGMII 2.5G mode enable\n");
1208	else
1209	netif_dbg(adapter, drv, adapter->netdev,
1210	"SGMII 1G mode enable\n");
1211
1212	/* SGMII/1000/2500BASE-X PCS power down */
1213	mii_ctl = lan743x_sgmii_read(adapter, MDIO_MMD_VEND2, MII_BMCR);
1214	if (mii_ctl < 0)
1215	return mii_ctl;
1216
1217	mii_ctl |= BMCR_PDOWN;
1218	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR, val: mii_ctl);
1219	if (ret < 0)
1220	return ret;
1221
1222	ret = lan743x_pcs_seq_state(adapter, PCS_POWER_STATE_DOWN);
1223	if (ret < 0)
1224	return ret;
1225
1226	/* SGMII/1000/2500BASE-X PCS power up */
1227	mii_ctl &= ~BMCR_PDOWN;
1228	ret = lan743x_sgmii_write(adapter, MDIO_MMD_VEND2, MII_BMCR, val: mii_ctl);
1229	if (ret < 0)
1230	return ret;
1231
1232	ret = lan743x_pcs_seq_state(adapter, PCS_POWER_STATE_UP);
1233	if (ret < 0)
1234	return ret;
1235
1236	return 0;
1237	}
1238
1239	static void lan743x_mac_set_address(struct lan743x_adapter *adapter,
1240	u8 *addr)
1241	{
1242	u32 addr_lo, addr_hi;
1243
1244	addr_lo = addr[0] |
1245	addr[1] << 8 |
1246	addr[2] << 16 |
1247	addr[3] << 24;
1248	addr_hi = addr[4] |
1249	addr[5] << 8;
1250	lan743x_csr_write(adapter, MAC_RX_ADDRL, data: addr_lo);
1251	lan743x_csr_write(adapter, MAC_RX_ADDRH, data: addr_hi);
1252
1253	ether_addr_copy(dst: adapter->mac_address, src: addr);
1254	netif_info(adapter, drv, adapter->netdev,
1255	"MAC address set to %pM\n", addr);
1256	}
1257
1258	static int lan743x_mac_init(struct lan743x_adapter *adapter)
1259	{
1260	bool mac_address_valid = true;
1261	struct net_device *netdev;
1262	u32 mac_addr_hi = 0;
1263	u32 mac_addr_lo = 0;
1264	u32 data;
1265
1266	netdev = adapter->netdev;
1267
1268	/* disable auto duplex, and speed detection. Phylib does that */
1269	data = lan743x_csr_read(adapter, MAC_CR);
1270	data &= ~(MAC_CR_ADD_ | MAC_CR_ASD_);
1271	data |= MAC_CR_CNTR_RST_;
1272	lan743x_csr_write(adapter, MAC_CR, data);
1273
1274	if (!is_valid_ether_addr(addr: adapter->mac_address)) {
1275	mac_addr_hi = lan743x_csr_read(adapter, MAC_RX_ADDRH);
1276	mac_addr_lo = lan743x_csr_read(adapter, MAC_RX_ADDRL);
1277	adapter->mac_address[0] = mac_addr_lo & 0xFF;
1278	adapter->mac_address[1] = (mac_addr_lo >> 8) & 0xFF;
1279	adapter->mac_address[2] = (mac_addr_lo >> 16) & 0xFF;
1280	adapter->mac_address[3] = (mac_addr_lo >> 24) & 0xFF;
1281	adapter->mac_address[4] = mac_addr_hi & 0xFF;
1282	adapter->mac_address[5] = (mac_addr_hi >> 8) & 0xFF;
1283
1284	if (((mac_addr_hi & 0x0000FFFF) == 0x0000FFFF) &&
1285	mac_addr_lo == 0xFFFFFFFF) {
1286	mac_address_valid = false;
1287	} else if (!is_valid_ether_addr(addr: adapter->mac_address)) {
1288	mac_address_valid = false;
1289	}
1290
1291	if (!mac_address_valid)
1292	eth_random_addr(addr: adapter->mac_address);
1293	}
1294	lan743x_mac_set_address(adapter, addr: adapter->mac_address);
1295	eth_hw_addr_set(dev: netdev, addr: adapter->mac_address);
1296
1297	return 0;
1298	}
1299
1300	static int lan743x_mac_open(struct lan743x_adapter *adapter)
1301	{
1302	u32 temp;
1303
1304	temp = lan743x_csr_read(adapter, MAC_RX);
1305	lan743x_csr_write(adapter, MAC_RX, data: temp | MAC_RX_RXEN_);
1306	temp = lan743x_csr_read(adapter, MAC_TX);
1307	lan743x_csr_write(adapter, MAC_TX, data: temp | MAC_TX_TXEN_);
1308	return 0;
1309	}
1310
1311	static void lan743x_mac_close(struct lan743x_adapter *adapter)
1312	{
1313	u32 temp;
1314
1315	temp = lan743x_csr_read(adapter, MAC_TX);
1316	temp &= ~MAC_TX_TXEN_;
1317	lan743x_csr_write(adapter, MAC_TX, data: temp);
1318	lan743x_csr_wait_for_bit(adapter, MAC_TX, MAC_TX_TXD_,
1319	target_value: 1, usleep_min: 1000, usleep_max: 20000, count: 100);
1320
1321	temp = lan743x_csr_read(adapter, MAC_RX);
1322	temp &= ~MAC_RX_RXEN_;
1323	lan743x_csr_write(adapter, MAC_RX, data: temp);
1324	lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
1325	target_value: 1, usleep_min: 1000, usleep_max: 20000, count: 100);
1326	}
1327
1328	void lan743x_mac_flow_ctrl_set_enables(struct lan743x_adapter *adapter,
1329	bool tx_enable, bool rx_enable)
1330	{
1331	u32 flow_setting = 0;
1332
1333	/* set maximum pause time because when fifo space frees
1334	* up a zero value pause frame will be sent to release the pause
1335	*/
1336	flow_setting = MAC_FLOW_CR_FCPT_MASK_;
1337	if (tx_enable)
1338	flow_setting |= MAC_FLOW_CR_TX_FCEN_;
1339	if (rx_enable)
1340	flow_setting |= MAC_FLOW_CR_RX_FCEN_;
1341	lan743x_csr_write(adapter, MAC_FLOW, data: flow_setting);
1342	}
1343
1344	static int lan743x_mac_set_mtu(struct lan743x_adapter *adapter, int new_mtu)
1345	{
1346	int enabled = 0;
1347	u32 mac_rx = 0;
1348
1349	mac_rx = lan743x_csr_read(adapter, MAC_RX);
1350	if (mac_rx & MAC_RX_RXEN_) {
1351	enabled = 1;
1352	if (mac_rx & MAC_RX_RXD_) {
1353	lan743x_csr_write(adapter, MAC_RX, data: mac_rx);
1354	mac_rx &= ~MAC_RX_RXD_;
1355	}
1356	mac_rx &= ~MAC_RX_RXEN_;
1357	lan743x_csr_write(adapter, MAC_RX, data: mac_rx);
1358	lan743x_csr_wait_for_bit(adapter, MAC_RX, MAC_RX_RXD_,
1359	target_value: 1, usleep_min: 1000, usleep_max: 20000, count: 100);
1360	lan743x_csr_write(adapter, MAC_RX, data: mac_rx | MAC_RX_RXD_);
1361	}
1362
1363	mac_rx &= ~(MAC_RX_MAX_SIZE_MASK_);
1364	mac_rx |= (((new_mtu + ETH_HLEN + ETH_FCS_LEN)
1365	<< MAC_RX_MAX_SIZE_SHIFT_) & MAC_RX_MAX_SIZE_MASK_);
1366	lan743x_csr_write(adapter, MAC_RX, data: mac_rx);
1367
1368	if (enabled) {
1369	mac_rx |= MAC_RX_RXEN_;
1370	lan743x_csr_write(adapter, MAC_RX, data: mac_rx);
1371	}
1372	return 0;
1373	}
1374
1375	/* PHY */
1376	static int lan743x_phy_reset(struct lan743x_adapter *adapter)
1377	{
1378	u32 data;
1379
1380	/* Only called with in probe, and before mdiobus_register */
1381
1382	data = lan743x_csr_read(adapter, PMT_CTL);
1383	data |= PMT_CTL_ETH_PHY_RST_;
1384	lan743x_csr_write(adapter, PMT_CTL, data);
1385
1386	return readx_poll_timeout(LAN743X_CSR_READ_OP, PMT_CTL, data,
1387	(!(data & PMT_CTL_ETH_PHY_RST_) &&
1388	(data & PMT_CTL_READY_)),
1389	50000, 1000000);
1390	}
1391
1392	static void lan743x_phy_update_flowcontrol(struct lan743x_adapter *adapter,
1393	u16 local_adv, u16 remote_adv)
1394	{
1395	struct lan743x_phy *phy = &adapter->phy;
1396	u8 cap;
1397
1398	if (phy->fc_autoneg)
1399	cap = mii_resolve_flowctrl_fdx(lcladv: local_adv, rmtadv: remote_adv);
1400	else
1401	cap = phy->fc_request_control;
1402
1403	lan743x_mac_flow_ctrl_set_enables(adapter,
1404	tx_enable: cap & FLOW_CTRL_TX,
1405	rx_enable: cap & FLOW_CTRL_RX);
1406	}
1407
1408	static int lan743x_phy_init(struct lan743x_adapter *adapter)
1409	{
1410	return lan743x_phy_reset(adapter);
1411	}
1412
1413	static void lan743x_phy_link_status_change(struct net_device *netdev)
1414	{
1415	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
1416	struct phy_device *phydev = netdev->phydev;
1417	u32 data;
1418
1419	phy_print_status(phydev);
1420	if (phydev->state == PHY_RUNNING) {
1421	int remote_advertisement = 0;
1422	int local_advertisement = 0;
1423
1424	data = lan743x_csr_read(adapter, MAC_CR);
1425
1426	/* set duplex mode */
1427	if (phydev->duplex)
1428	data |= MAC_CR_DPX_;
1429	else
1430	data &= ~MAC_CR_DPX_;
1431
1432	/* set bus speed */
1433	switch (phydev->speed) {
1434	case SPEED_10:
1435	data &= ~MAC_CR_CFG_H_;
1436	data &= ~MAC_CR_CFG_L_;
1437	break;
1438	case SPEED_100:
1439	data &= ~MAC_CR_CFG_H_;
1440	data |= MAC_CR_CFG_L_;
1441	break;
1442	case SPEED_1000:
1443	data |= MAC_CR_CFG_H_;
1444	data &= ~MAC_CR_CFG_L_;
1445	break;
1446	case SPEED_2500:
1447	data |= MAC_CR_CFG_H_;
1448	data |= MAC_CR_CFG_L_;
1449	break;
1450	}
1451	lan743x_csr_write(adapter, MAC_CR, data);
1452
1453	local_advertisement =
1454	linkmode_adv_to_mii_adv_t(advertising: phydev->advertising);
1455	remote_advertisement =
1456	linkmode_adv_to_mii_adv_t(advertising: phydev->lp_advertising);
1457
1458	lan743x_phy_update_flowcontrol(adapter, local_adv: local_advertisement,
1459	remote_adv: remote_advertisement);
1460	lan743x_ptp_update_latency(adapter, link_speed: phydev->speed);
1461	if (phydev->interface == PHY_INTERFACE_MODE_SGMII ||
1462	phydev->interface == PHY_INTERFACE_MODE_1000BASEX ||
1463	phydev->interface == PHY_INTERFACE_MODE_2500BASEX)
1464	lan743x_sgmii_config(adapter);
1465	}
1466	}
1467
1468	static void lan743x_phy_close(struct lan743x_adapter *adapter)
1469	{
1470	struct net_device *netdev = adapter->netdev;
1471	struct phy_device *phydev = netdev->phydev;
1472
1473	phy_stop(phydev: netdev->phydev);
1474	phy_disconnect(phydev: netdev->phydev);
1475
1476	/* using phydev here as phy_disconnect NULLs netdev->phydev */
1477	if (phy_is_pseudo_fixed_link(phydev))
1478	fixed_phy_unregister(phydev);
1479
1480	}
1481
1482	static void lan743x_phy_interface_select(struct lan743x_adapter *adapter)
1483	{
1484	u32 id_rev;
1485	u32 data;
1486
1487	data = lan743x_csr_read(adapter, MAC_CR);
1488	id_rev = adapter->csr.id_rev & ID_REV_ID_MASK_;
1489
1490	if (adapter->is_pci11x1x && adapter->is_sgmii_en)
1491	adapter->phy_interface = PHY_INTERFACE_MODE_SGMII;
1492	else if (id_rev == ID_REV_ID_LAN7430_)
1493	adapter->phy_interface = PHY_INTERFACE_MODE_GMII;
1494	else if ((id_rev == ID_REV_ID_LAN7431_) && (data & MAC_CR_MII_EN_))
1495	adapter->phy_interface = PHY_INTERFACE_MODE_MII;
1496	else
1497	adapter->phy_interface = PHY_INTERFACE_MODE_RGMII;
1498	}
1499
1500	static int lan743x_phy_open(struct lan743x_adapter *adapter)
1501	{
1502	struct net_device *netdev = adapter->netdev;
1503	struct lan743x_phy *phy = &adapter->phy;
1504	struct fixed_phy_status fphy_status = {
1505	.link = 1,
1506	.speed = SPEED_1000,
1507	.duplex = DUPLEX_FULL,
1508	};
1509	struct phy_device *phydev;
1510	int ret = -EIO;
1511
1512	/* try devicetree phy, or fixed link */
1513	phydev = of_phy_get_and_connect(dev: netdev, np: adapter->pdev->dev.of_node,
1514	hndlr: lan743x_phy_link_status_change);
1515
1516	if (!phydev) {
1517	/* try internal phy */
1518	phydev = phy_find_first(bus: adapter->mdiobus);
1519	if (!phydev) {
1520	if ((adapter->csr.id_rev & ID_REV_ID_MASK_) ==
1521	ID_REV_ID_LAN7431_) {
1522	phydev = fixed_phy_register(PHY_POLL,
1523	status: &fphy_status, NULL);
1524	if (IS_ERR(ptr: phydev)) {
1525	netdev_err(dev: netdev, format: "No PHY/fixed_PHY found\n");
1526	return PTR_ERR(ptr: phydev);
1527	}
1528	} else {
1529	goto return_error;
1530	}
1531	}
1532
1533	lan743x_phy_interface_select(adapter);
1534
1535	ret = phy_connect_direct(dev: netdev, phydev,
1536	handler: lan743x_phy_link_status_change,
1537	interface: adapter->phy_interface);
1538	if (ret)
1539	goto return_error;
1540	}
1541
1542	/* MAC doesn't support 1000T Half */
1543	phy_remove_link_mode(phydev, link_mode: ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1544
1545	/* support both flow controls */
1546	phy_support_asym_pause(phydev);
1547	phy->fc_request_control = (FLOW_CTRL_RX | FLOW_CTRL_TX);
1548	phy->fc_autoneg = phydev->autoneg;
1549
1550	phy_start(phydev);
1551	phy_start_aneg(phydev);
1552	phy_attached_info(phydev);
1553	return 0;
1554
1555	return_error:
1556	return ret;
1557	}
1558
1559	static void lan743x_rfe_open(struct lan743x_adapter *adapter)
1560	{
1561	lan743x_csr_write(adapter, RFE_RSS_CFG,
1562	RFE_RSS_CFG_UDP_IPV6_EX_ |
1563	RFE_RSS_CFG_TCP_IPV6_EX_ |
1564	RFE_RSS_CFG_IPV6_EX_ |
1565	RFE_RSS_CFG_UDP_IPV6_ |
1566	RFE_RSS_CFG_TCP_IPV6_ |
1567	RFE_RSS_CFG_IPV6_ |
1568	RFE_RSS_CFG_UDP_IPV4_ |
1569	RFE_RSS_CFG_TCP_IPV4_ |
1570	RFE_RSS_CFG_IPV4_ |
1571	RFE_RSS_CFG_VALID_HASH_BITS_ |
1572	RFE_RSS_CFG_RSS_QUEUE_ENABLE_ |
1573	RFE_RSS_CFG_RSS_HASH_STORE_ |
1574	RFE_RSS_CFG_RSS_ENABLE_);
1575	}
1576
1577	static void lan743x_rfe_update_mac_address(struct lan743x_adapter *adapter)
1578	{
1579	u8 *mac_addr;
1580	u32 mac_addr_hi = 0;
1581	u32 mac_addr_lo = 0;
1582
1583	/* Add mac address to perfect Filter */
1584	mac_addr = adapter->mac_address;
1585	mac_addr_lo = ((((u32)(mac_addr[0])) << 0) |
1586	(((u32)(mac_addr[1])) << 8) |
1587	(((u32)(mac_addr[2])) << 16) |
1588	(((u32)(mac_addr[3])) << 24));
1589	mac_addr_hi = ((((u32)(mac_addr[4])) << 0) |
1590	(((u32)(mac_addr[5])) << 8));
1591
1592	lan743x_csr_write(adapter, RFE_ADDR_FILT_LO(0), data: mac_addr_lo);
1593	lan743x_csr_write(adapter, RFE_ADDR_FILT_HI(0),
1594	data: mac_addr_hi | RFE_ADDR_FILT_HI_VALID_);
1595	}
1596
1597	static void lan743x_rfe_set_multicast(struct lan743x_adapter *adapter)
1598	{
1599	struct net_device *netdev = adapter->netdev;
1600	u32 hash_table[DP_SEL_VHF_HASH_LEN];
1601	u32 rfctl;
1602	u32 data;
1603
1604	rfctl = lan743x_csr_read(adapter, RFE_CTL);
1605	rfctl &= ~(RFE_CTL_AU_ | RFE_CTL_AM_ |
1606	RFE_CTL_DA_PERFECT_ | RFE_CTL_MCAST_HASH_);
1607	rfctl |= RFE_CTL_AB_;
1608	if (netdev->flags & IFF_PROMISC) {
1609	rfctl |= RFE_CTL_AM_ | RFE_CTL_AU_;
1610	} else {
1611	if (netdev->flags & IFF_ALLMULTI)
1612	rfctl |= RFE_CTL_AM_;
1613	}
1614
1615	if (netdev->features & NETIF_F_RXCSUM)
1616	rfctl |= RFE_CTL_IP_COE_ | RFE_CTL_TCP_UDP_COE_;
1617
1618	memset(hash_table, 0, DP_SEL_VHF_HASH_LEN * sizeof(u32));
1619	if (netdev_mc_count(netdev)) {
1620	struct netdev_hw_addr *ha;
1621	int i;
1622
1623	rfctl |= RFE_CTL_DA_PERFECT_;
1624	i = 1;
1625	netdev_for_each_mc_addr(ha, netdev) {
1626	/* set first 32 into Perfect Filter */
1627	if (i < 33) {
1628	lan743x_csr_write(adapter,
1629	RFE_ADDR_FILT_HI(i), data: 0);
1630	data = ha->addr[3];
1631	data = ha->addr[2] | (data << 8);
1632	data = ha->addr[1] | (data << 8);
1633	data = ha->addr[0] | (data << 8);
1634	lan743x_csr_write(adapter,
1635	RFE_ADDR_FILT_LO(i), data);
1636	data = ha->addr[5];
1637	data = ha->addr[4] | (data << 8);
1638	data |= RFE_ADDR_FILT_HI_VALID_;
1639	lan743x_csr_write(adapter,
1640	RFE_ADDR_FILT_HI(i), data);
1641	} else {
1642	u32 bitnum = (ether_crc(ETH_ALEN, ha->addr) >>
1643	23) & 0x1FF;
1644	hash_table[bitnum / 32] |= (1 << (bitnum % 32));
1645	rfctl |= RFE_CTL_MCAST_HASH_;
1646	}
1647	i++;
1648	}
1649	}
1650
1651	lan743x_dp_write(adapter, DP_SEL_RFE_RAM,
1652	DP_SEL_VHF_VLAN_LEN,
1653	DP_SEL_VHF_HASH_LEN, buf: hash_table);
1654	lan743x_csr_write(adapter, RFE_CTL, data: rfctl);
1655	}
1656
1657	static int lan743x_dmac_init(struct lan743x_adapter *adapter)
1658	{
1659	u32 data = 0;
1660
1661	lan743x_csr_write(adapter, DMAC_CMD, DMAC_CMD_SWR_);
1662	lan743x_csr_wait_for_bit(adapter, DMAC_CMD, DMAC_CMD_SWR_,
1663	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
1664	switch (DEFAULT_DMA_DESCRIPTOR_SPACING) {
1665	case DMA_DESCRIPTOR_SPACING_16:
1666	data = DMAC_CFG_MAX_DSPACE_16_;
1667	break;
1668	case DMA_DESCRIPTOR_SPACING_32:
1669	data = DMAC_CFG_MAX_DSPACE_32_;
1670	break;
1671	case DMA_DESCRIPTOR_SPACING_64:
1672	data = DMAC_CFG_MAX_DSPACE_64_;
1673	break;
1674	case DMA_DESCRIPTOR_SPACING_128:
1675	data = DMAC_CFG_MAX_DSPACE_128_;
1676	break;
1677	default:
1678	return -EPERM;
1679	}
1680	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
1681	data |= DMAC_CFG_COAL_EN_;
1682	data |= DMAC_CFG_CH_ARB_SEL_RX_HIGH_;
1683	data |= DMAC_CFG_MAX_READ_REQ_SET_(6);
1684	lan743x_csr_write(adapter, DMAC_CFG, data);
1685	data = DMAC_COAL_CFG_TIMER_LIMIT_SET_(1);
1686	data |= DMAC_COAL_CFG_TIMER_TX_START_;
1687	data |= DMAC_COAL_CFG_FLUSH_INTS_;
1688	data |= DMAC_COAL_CFG_INT_EXIT_COAL_;
1689	data |= DMAC_COAL_CFG_CSR_EXIT_COAL_;
1690	data |= DMAC_COAL_CFG_TX_THRES_SET_(0x0A);
1691	data |= DMAC_COAL_CFG_RX_THRES_SET_(0x0C);
1692	lan743x_csr_write(adapter, DMAC_COAL_CFG, data);
1693	data = DMAC_OBFF_TX_THRES_SET_(0x08);
1694	data |= DMAC_OBFF_RX_THRES_SET_(0x0A);
1695	lan743x_csr_write(adapter, DMAC_OBFF_CFG, data);
1696	return 0;
1697	}
1698
1699	static int lan743x_dmac_tx_get_state(struct lan743x_adapter *adapter,
1700	int tx_channel)
1701	{
1702	u32 dmac_cmd = 0;
1703
1704	dmac_cmd = lan743x_csr_read(adapter, DMAC_CMD);
1705	return DMAC_CHANNEL_STATE_SET((dmac_cmd &
1706	DMAC_CMD_START_T_(tx_channel)),
1707	(dmac_cmd &
1708	DMAC_CMD_STOP_T_(tx_channel)));
1709	}
1710
1711	static int lan743x_dmac_tx_wait_till_stopped(struct lan743x_adapter *adapter,
1712	int tx_channel)
1713	{
1714	int timeout = 100;
1715	int result = 0;
1716
1717	while (timeout &&
1718	((result = lan743x_dmac_tx_get_state(adapter, tx_channel)) ==
1719	DMAC_CHANNEL_STATE_STOP_PENDING)) {
1720	usleep_range(min: 1000, max: 20000);
1721	timeout--;
1722	}
1723	if (result == DMAC_CHANNEL_STATE_STOP_PENDING)
1724	result = -ENODEV;
1725	return result;
1726	}
1727
1728	static int lan743x_dmac_rx_get_state(struct lan743x_adapter *adapter,
1729	int rx_channel)
1730	{
1731	u32 dmac_cmd = 0;
1732
1733	dmac_cmd = lan743x_csr_read(adapter, DMAC_CMD);
1734	return DMAC_CHANNEL_STATE_SET((dmac_cmd &
1735	DMAC_CMD_START_R_(rx_channel)),
1736	(dmac_cmd &
1737	DMAC_CMD_STOP_R_(rx_channel)));
1738	}
1739
1740	static int lan743x_dmac_rx_wait_till_stopped(struct lan743x_adapter *adapter,
1741	int rx_channel)
1742	{
1743	int timeout = 100;
1744	int result = 0;
1745
1746	while (timeout &&
1747	((result = lan743x_dmac_rx_get_state(adapter, rx_channel)) ==
1748	DMAC_CHANNEL_STATE_STOP_PENDING)) {
1749	usleep_range(min: 1000, max: 20000);
1750	timeout--;
1751	}
1752	if (result == DMAC_CHANNEL_STATE_STOP_PENDING)
1753	result = -ENODEV;
1754	return result;
1755	}
1756
1757	static void lan743x_tx_release_desc(struct lan743x_tx *tx,
1758	int descriptor_index, bool cleanup)
1759	{
1760	struct lan743x_tx_buffer_info *buffer_info = NULL;
1761	struct lan743x_tx_descriptor *descriptor = NULL;
1762	u32 descriptor_type = 0;
1763	bool ignore_sync;
1764
1765	descriptor = &tx->ring_cpu_ptr[descriptor_index];
1766	buffer_info = &tx->buffer_info[descriptor_index];
1767	if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_ACTIVE))
1768	goto done;
1769
1770	descriptor_type = le32_to_cpu(descriptor->data0) &
1771	TX_DESC_DATA0_DTYPE_MASK_;
1772	if (descriptor_type == TX_DESC_DATA0_DTYPE_DATA_)
1773	goto clean_up_data_descriptor;
1774	else
1775	goto clear_active;
1776
1777	clean_up_data_descriptor:
1778	if (buffer_info->dma_ptr) {
1779	if (buffer_info->flags &
1780	TX_BUFFER_INFO_FLAG_SKB_FRAGMENT) {
1781	dma_unmap_page(&tx->adapter->pdev->dev,
1782	buffer_info->dma_ptr,
1783	buffer_info->buffer_length,
1784	DMA_TO_DEVICE);
1785	} else {
1786	dma_unmap_single(&tx->adapter->pdev->dev,
1787	buffer_info->dma_ptr,
1788	buffer_info->buffer_length,
1789	DMA_TO_DEVICE);
1790	}
1791	buffer_info->dma_ptr = 0;
1792	buffer_info->buffer_length = 0;
1793	}
1794	if (!buffer_info->skb)
1795	goto clear_active;
1796
1797	if (!(buffer_info->flags & TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED)) {
1798	dev_kfree_skb_any(skb: buffer_info->skb);
1799	goto clear_skb;
1800	}
1801
1802	if (cleanup) {
1803	lan743x_ptp_unrequest_tx_timestamp(adapter: tx->adapter);
1804	dev_kfree_skb_any(skb: buffer_info->skb);
1805	} else {
1806	ignore_sync = (buffer_info->flags &
1807	TX_BUFFER_INFO_FLAG_IGNORE_SYNC) != 0;
1808	lan743x_ptp_tx_timestamp_skb(adapter: tx->adapter,
1809	skb: buffer_info->skb, ignore_sync);
1810	}
1811
1812	clear_skb:
1813	buffer_info->skb = NULL;
1814
1815	clear_active:
1816	buffer_info->flags &= ~TX_BUFFER_INFO_FLAG_ACTIVE;
1817
1818	done:
1819	memset(buffer_info, 0, sizeof(*buffer_info));
1820	memset(descriptor, 0, sizeof(*descriptor));
1821	}
1822
1823	static int lan743x_tx_next_index(struct lan743x_tx *tx, int index)
1824	{
1825	return ((++index) % tx->ring_size);
1826	}
1827
1828	static void lan743x_tx_release_completed_descriptors(struct lan743x_tx *tx)
1829	{
1830	while (le32_to_cpu(*tx->head_cpu_ptr) != (tx->last_head)) {
1831	lan743x_tx_release_desc(tx, descriptor_index: tx->last_head, cleanup: false);
1832	tx->last_head = lan743x_tx_next_index(tx, index: tx->last_head);
1833	}
1834	}
1835
1836	static void lan743x_tx_release_all_descriptors(struct lan743x_tx *tx)
1837	{
1838	u32 original_head = 0;
1839
1840	original_head = tx->last_head;
1841	do {
1842	lan743x_tx_release_desc(tx, descriptor_index: tx->last_head, cleanup: true);
1843	tx->last_head = lan743x_tx_next_index(tx, index: tx->last_head);
1844	} while (tx->last_head != original_head);
1845	memset(tx->ring_cpu_ptr, 0,
1846	sizeof(*tx->ring_cpu_ptr) * (tx->ring_size));
1847	memset(tx->buffer_info, 0,
1848	sizeof(*tx->buffer_info) * (tx->ring_size));
1849	}
1850
1851	static int lan743x_tx_get_desc_cnt(struct lan743x_tx *tx,
1852	struct sk_buff *skb)
1853	{
1854	int result = 1; /* 1 for the main skb buffer */
1855	int nr_frags = 0;
1856
1857	if (skb_is_gso(skb))
1858	result++; /* requires an extension descriptor */
1859	nr_frags = skb_shinfo(skb)->nr_frags;
1860	result += nr_frags; /* 1 for each fragment buffer */
1861	return result;
1862	}
1863
1864	static int lan743x_tx_get_avail_desc(struct lan743x_tx *tx)
1865	{
1866	int last_head = tx->last_head;
1867	int last_tail = tx->last_tail;
1868
1869	if (last_tail >= last_head)
1870	return tx->ring_size - last_tail + last_head - 1;
1871	else
1872	return last_head - last_tail - 1;
1873	}
1874
1875	static void lan743x_rx_cfg_b_tstamp_config(struct lan743x_adapter *adapter,
1876	int rx_ts_config)
1877	{
1878	int channel_number;
1879	int index;
1880	u32 data;
1881
1882	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
1883	channel_number = adapter->rx[index].channel_number;
1884	data = lan743x_csr_read(adapter, RX_CFG_B(channel_number));
1885	data &= RX_CFG_B_TS_MASK_;
1886	data |= rx_ts_config;
1887	lan743x_csr_write(adapter, RX_CFG_B(channel_number),
1888	data);
1889	}
1890	}
1891
1892	int lan743x_rx_set_tstamp_mode(struct lan743x_adapter *adapter,
1893	int rx_filter)
1894	{
1895	u32 data;
1896
1897	switch (rx_filter) {
1898	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1899	lan743x_rx_cfg_b_tstamp_config(adapter,
1900	RX_CFG_B_TS_DESCR_EN_);
1901	data = lan743x_csr_read(adapter, PTP_RX_TS_CFG);
1902	data |= PTP_RX_TS_CFG_EVENT_MSGS_;
1903	lan743x_csr_write(adapter, PTP_RX_TS_CFG, data);
1904	break;
1905	case HWTSTAMP_FILTER_NONE:
1906	lan743x_rx_cfg_b_tstamp_config(adapter,
1907	RX_CFG_B_TS_NONE_);
1908	break;
1909	case HWTSTAMP_FILTER_ALL:
1910	lan743x_rx_cfg_b_tstamp_config(adapter,
1911	RX_CFG_B_TS_ALL_RX_);
1912	break;
1913	default:
1914	return -ERANGE;
1915	}
1916	return 0;
1917	}
1918
1919	void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx,
1920	bool enable_timestamping,
1921	bool enable_onestep_sync)
1922	{
1923	if (enable_timestamping)
1924	tx->ts_flags |= TX_TS_FLAG_TIMESTAMPING_ENABLED;
1925	else
1926	tx->ts_flags &= ~TX_TS_FLAG_TIMESTAMPING_ENABLED;
1927	if (enable_onestep_sync)
1928	tx->ts_flags |= TX_TS_FLAG_ONE_STEP_SYNC;
1929	else
1930	tx->ts_flags &= ~TX_TS_FLAG_ONE_STEP_SYNC;
1931	}
1932
1933	static int lan743x_tx_frame_start(struct lan743x_tx *tx,
1934	unsigned char *first_buffer,
1935	unsigned int first_buffer_length,
1936	unsigned int frame_length,
1937	bool time_stamp,
1938	bool check_sum)
1939	{
1940	/* called only from within lan743x_tx_xmit_frame.
1941	* assuming tx->ring_lock has already been acquired.
1942	*/
1943	struct lan743x_tx_descriptor *tx_descriptor = NULL;
1944	struct lan743x_tx_buffer_info *buffer_info = NULL;
1945	struct lan743x_adapter *adapter = tx->adapter;
1946	struct device *dev = &adapter->pdev->dev;
1947	dma_addr_t dma_ptr;
1948
1949	tx->frame_flags |= TX_FRAME_FLAG_IN_PROGRESS;
1950	tx->frame_first = tx->last_tail;
1951	tx->frame_tail = tx->frame_first;
1952
1953	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
1954	buffer_info = &tx->buffer_info[tx->frame_tail];
1955	dma_ptr = dma_map_single(dev, first_buffer, first_buffer_length,
1956	DMA_TO_DEVICE);
1957	if (dma_mapping_error(dev, dma_addr: dma_ptr))
1958	return -ENOMEM;
1959
1960	tx_descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(dma_ptr));
1961	tx_descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(dma_ptr));
1962	tx_descriptor->data3 = cpu_to_le32((frame_length << 16) &
1963	TX_DESC_DATA3_FRAME_LENGTH_MSS_MASK_);
1964
1965	buffer_info->skb = NULL;
1966	buffer_info->dma_ptr = dma_ptr;
1967	buffer_info->buffer_length = first_buffer_length;
1968	buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
1969
1970	tx->frame_data0 = (first_buffer_length &
1971	TX_DESC_DATA0_BUF_LENGTH_MASK_) |
1972	TX_DESC_DATA0_DTYPE_DATA_ |
1973	TX_DESC_DATA0_FS_ |
1974	TX_DESC_DATA0_FCS_;
1975	if (time_stamp)
1976	tx->frame_data0 |= TX_DESC_DATA0_TSE_;
1977
1978	if (check_sum)
1979	tx->frame_data0 |= TX_DESC_DATA0_ICE_ |
1980	TX_DESC_DATA0_IPE_ |
1981	TX_DESC_DATA0_TPE_;
1982
1983	/* data0 will be programmed in one of other frame assembler functions */
1984	return 0;
1985	}
1986
1987	static void lan743x_tx_frame_add_lso(struct lan743x_tx *tx,
1988	unsigned int frame_length,
1989	int nr_frags)
1990	{
1991	/* called only from within lan743x_tx_xmit_frame.
1992	* assuming tx->ring_lock has already been acquired.
1993	*/
1994	struct lan743x_tx_descriptor *tx_descriptor = NULL;
1995	struct lan743x_tx_buffer_info *buffer_info = NULL;
1996
1997	/* wrap up previous descriptor */
1998	tx->frame_data0 |= TX_DESC_DATA0_EXT_;
1999	if (nr_frags <= 0) {
2000	tx->frame_data0 |= TX_DESC_DATA0_LS_;
2001	tx->frame_data0 |= TX_DESC_DATA0_IOC_;
2002	}
2003	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2004	tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
2005
2006	/* move to next descriptor */
2007	tx->frame_tail = lan743x_tx_next_index(tx, index: tx->frame_tail);
2008	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2009	buffer_info = &tx->buffer_info[tx->frame_tail];
2010
2011	/* add extension descriptor */
2012	tx_descriptor->data1 = 0;
2013	tx_descriptor->data2 = 0;
2014	tx_descriptor->data3 = 0;
2015
2016	buffer_info->skb = NULL;
2017	buffer_info->dma_ptr = 0;
2018	buffer_info->buffer_length = 0;
2019	buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
2020
2021	tx->frame_data0 = (frame_length & TX_DESC_DATA0_EXT_PAY_LENGTH_MASK_) |
2022	TX_DESC_DATA0_DTYPE_EXT_ |
2023	TX_DESC_DATA0_EXT_LSO_;
2024
2025	/* data0 will be programmed in one of other frame assembler functions */
2026	}
2027
2028	static int lan743x_tx_frame_add_fragment(struct lan743x_tx *tx,
2029	const skb_frag_t *fragment,
2030	unsigned int frame_length)
2031	{
2032	/* called only from within lan743x_tx_xmit_frame
2033	* assuming tx->ring_lock has already been acquired
2034	*/
2035	struct lan743x_tx_descriptor *tx_descriptor = NULL;
2036	struct lan743x_tx_buffer_info *buffer_info = NULL;
2037	struct lan743x_adapter *adapter = tx->adapter;
2038	struct device *dev = &adapter->pdev->dev;
2039	unsigned int fragment_length = 0;
2040	dma_addr_t dma_ptr;
2041
2042	fragment_length = skb_frag_size(frag: fragment);
2043	if (!fragment_length)
2044	return 0;
2045
2046	/* wrap up previous descriptor */
2047	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2048	tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
2049
2050	/* move to next descriptor */
2051	tx->frame_tail = lan743x_tx_next_index(tx, index: tx->frame_tail);
2052	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2053	buffer_info = &tx->buffer_info[tx->frame_tail];
2054	dma_ptr = skb_frag_dma_map(dev, frag: fragment,
2055	offset: 0, size: fragment_length,
2056	dir: DMA_TO_DEVICE);
2057	if (dma_mapping_error(dev, dma_addr: dma_ptr)) {
2058	int desc_index;
2059
2060	/* cleanup all previously setup descriptors */
2061	desc_index = tx->frame_first;
2062	while (desc_index != tx->frame_tail) {
2063	lan743x_tx_release_desc(tx, descriptor_index: desc_index, cleanup: true);
2064	desc_index = lan743x_tx_next_index(tx, index: desc_index);
2065	}
2066	dma_wmb();
2067	tx->frame_flags &= ~TX_FRAME_FLAG_IN_PROGRESS;
2068	tx->frame_first = 0;
2069	tx->frame_data0 = 0;
2070	tx->frame_tail = 0;
2071	return -ENOMEM;
2072	}
2073
2074	tx_descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(dma_ptr));
2075	tx_descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(dma_ptr));
2076	tx_descriptor->data3 = cpu_to_le32((frame_length << 16) &
2077	TX_DESC_DATA3_FRAME_LENGTH_MSS_MASK_);
2078
2079	buffer_info->skb = NULL;
2080	buffer_info->dma_ptr = dma_ptr;
2081	buffer_info->buffer_length = fragment_length;
2082	buffer_info->flags |= TX_BUFFER_INFO_FLAG_ACTIVE;
2083	buffer_info->flags |= TX_BUFFER_INFO_FLAG_SKB_FRAGMENT;
2084
2085	tx->frame_data0 = (fragment_length & TX_DESC_DATA0_BUF_LENGTH_MASK_) |
2086	TX_DESC_DATA0_DTYPE_DATA_ |
2087	TX_DESC_DATA0_FCS_;
2088
2089	/* data0 will be programmed in one of other frame assembler functions */
2090	return 0;
2091	}
2092
2093	static void lan743x_tx_frame_end(struct lan743x_tx *tx,
2094	struct sk_buff *skb,
2095	bool time_stamp,
2096	bool ignore_sync)
2097	{
2098	/* called only from within lan743x_tx_xmit_frame
2099	* assuming tx->ring_lock has already been acquired
2100	*/
2101	struct lan743x_tx_descriptor *tx_descriptor = NULL;
2102	struct lan743x_tx_buffer_info *buffer_info = NULL;
2103	struct lan743x_adapter *adapter = tx->adapter;
2104	u32 tx_tail_flags = 0;
2105
2106	/* wrap up previous descriptor */
2107	if ((tx->frame_data0 & TX_DESC_DATA0_DTYPE_MASK_) ==
2108	TX_DESC_DATA0_DTYPE_DATA_) {
2109	tx->frame_data0 |= TX_DESC_DATA0_LS_;
2110	tx->frame_data0 |= TX_DESC_DATA0_IOC_;
2111	}
2112
2113	tx_descriptor = &tx->ring_cpu_ptr[tx->frame_tail];
2114	buffer_info = &tx->buffer_info[tx->frame_tail];
2115	buffer_info->skb = skb;
2116	if (time_stamp)
2117	buffer_info->flags |= TX_BUFFER_INFO_FLAG_TIMESTAMP_REQUESTED;
2118	if (ignore_sync)
2119	buffer_info->flags |= TX_BUFFER_INFO_FLAG_IGNORE_SYNC;
2120
2121	tx_descriptor->data0 = cpu_to_le32(tx->frame_data0);
2122	tx->frame_tail = lan743x_tx_next_index(tx, index: tx->frame_tail);
2123	tx->last_tail = tx->frame_tail;
2124
2125	dma_wmb();
2126
2127	if (tx->vector_flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET)
2128	tx_tail_flags |= TX_TAIL_SET_TOP_INT_VEC_EN_;
2129	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET)
2130	tx_tail_flags |= TX_TAIL_SET_DMAC_INT_EN_ |
2131	TX_TAIL_SET_TOP_INT_EN_;
2132
2133	lan743x_csr_write(adapter, TX_TAIL(tx->channel_number),
2134	data: tx_tail_flags | tx->frame_tail);
2135	tx->frame_flags &= ~TX_FRAME_FLAG_IN_PROGRESS;
2136	}
2137
2138	static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx,
2139	struct sk_buff *skb)
2140	{
2141	int required_number_of_descriptors = 0;
2142	unsigned int start_frame_length = 0;
2143	netdev_tx_t retval = NETDEV_TX_OK;
2144	unsigned int frame_length = 0;
2145	unsigned int head_length = 0;
2146	unsigned long irq_flags = 0;
2147	bool do_timestamp = false;
2148	bool ignore_sync = false;
2149	struct netdev_queue *txq;
2150	int nr_frags = 0;
2151	bool gso = false;
2152	int j;
2153
2154	required_number_of_descriptors = lan743x_tx_get_desc_cnt(tx, skb);
2155
2156	spin_lock_irqsave(&tx->ring_lock, irq_flags);
2157	if (required_number_of_descriptors >
2158	lan743x_tx_get_avail_desc(tx)) {
2159	if (required_number_of_descriptors > (tx->ring_size - 1)) {
2160	dev_kfree_skb_irq(skb);
2161	} else {
2162	/* save how many descriptors we needed to restart the queue */
2163	tx->rqd_descriptors = required_number_of_descriptors;
2164	retval = NETDEV_TX_BUSY;
2165	txq = netdev_get_tx_queue(dev: tx->adapter->netdev,
2166	index: tx->channel_number);
2167	netif_tx_stop_queue(dev_queue: txq);
2168	}
2169	goto unlock;
2170	}
2171
2172	/* space available, transmit skb */
2173	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2174	(tx->ts_flags & TX_TS_FLAG_TIMESTAMPING_ENABLED) &&
2175	(lan743x_ptp_request_tx_timestamp(adapter: tx->adapter))) {
2176	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2177	do_timestamp = true;
2178	if (tx->ts_flags & TX_TS_FLAG_ONE_STEP_SYNC)
2179	ignore_sync = true;
2180	}
2181	head_length = skb_headlen(skb);
2182	frame_length = skb_pagelen(skb);
2183	nr_frags = skb_shinfo(skb)->nr_frags;
2184	start_frame_length = frame_length;
2185	gso = skb_is_gso(skb);
2186	if (gso) {
2187	start_frame_length = max(skb_shinfo(skb)->gso_size,
2188	(unsigned short)8);
2189	}
2190
2191	if (lan743x_tx_frame_start(tx,
2192	first_buffer: skb->data, first_buffer_length: head_length,
2193	frame_length: start_frame_length,
2194	time_stamp: do_timestamp,
2195	check_sum: skb->ip_summed == CHECKSUM_PARTIAL)) {
2196	dev_kfree_skb_irq(skb);
2197	goto unlock;
2198	}
2199	tx->frame_count++;
2200
2201	if (gso)
2202	lan743x_tx_frame_add_lso(tx, frame_length, nr_frags);
2203
2204	if (nr_frags <= 0)
2205	goto finish;
2206
2207	for (j = 0; j < nr_frags; j++) {
2208	const skb_frag_t *frag = &(skb_shinfo(skb)->frags[j]);
2209
2210	if (lan743x_tx_frame_add_fragment(tx, fragment: frag, frame_length)) {
2211	/* upon error no need to call
2212	* lan743x_tx_frame_end
2213	* frame assembler clean up was performed inside
2214	* lan743x_tx_frame_add_fragment
2215	*/
2216	dev_kfree_skb_irq(skb);
2217	goto unlock;
2218	}
2219	}
2220
2221	finish:
2222	lan743x_tx_frame_end(tx, skb, time_stamp: do_timestamp, ignore_sync);
2223
2224	unlock:
2225	spin_unlock_irqrestore(lock: &tx->ring_lock, flags: irq_flags);
2226	return retval;
2227	}
2228
2229	static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight)
2230	{
2231	struct lan743x_tx *tx = container_of(napi, struct lan743x_tx, napi);
2232	struct lan743x_adapter *adapter = tx->adapter;
2233	unsigned long irq_flags = 0;
2234	struct netdev_queue *txq;
2235	u32 ioc_bit = 0;
2236
2237	ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
2238	lan743x_csr_read(adapter, DMAC_INT_STS);
2239	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C)
2240	lan743x_csr_write(adapter, DMAC_INT_STS, data: ioc_bit);
2241	spin_lock_irqsave(&tx->ring_lock, irq_flags);
2242
2243	/* clean up tx ring */
2244	lan743x_tx_release_completed_descriptors(tx);
2245	txq = netdev_get_tx_queue(dev: adapter->netdev, index: tx->channel_number);
2246	if (netif_tx_queue_stopped(dev_queue: txq)) {
2247	if (tx->rqd_descriptors) {
2248	if (tx->rqd_descriptors <=
2249	lan743x_tx_get_avail_desc(tx)) {
2250	tx->rqd_descriptors = 0;
2251	netif_tx_wake_queue(dev_queue: txq);
2252	}
2253	} else {
2254	netif_tx_wake_queue(dev_queue: txq);
2255	}
2256	}
2257	spin_unlock_irqrestore(lock: &tx->ring_lock, flags: irq_flags);
2258
2259	if (!napi_complete(n: napi))
2260	goto done;
2261
2262	/* enable isr */
2263	lan743x_csr_write(adapter, INT_EN_SET,
2264	INT_BIT_DMA_TX_(tx->channel_number));
2265	lan743x_csr_read(adapter, INT_STS);
2266
2267	done:
2268	return 0;
2269	}
2270
2271	static void lan743x_tx_ring_cleanup(struct lan743x_tx *tx)
2272	{
2273	if (tx->head_cpu_ptr) {
2274	dma_free_coherent(dev: &tx->adapter->pdev->dev,
2275	size: sizeof(*tx->head_cpu_ptr), cpu_addr: tx->head_cpu_ptr,
2276	dma_handle: tx->head_dma_ptr);
2277	tx->head_cpu_ptr = NULL;
2278	tx->head_dma_ptr = 0;
2279	}
2280	kfree(objp: tx->buffer_info);
2281	tx->buffer_info = NULL;
2282
2283	if (tx->ring_cpu_ptr) {
2284	dma_free_coherent(dev: &tx->adapter->pdev->dev,
2285	size: tx->ring_allocation_size, cpu_addr: tx->ring_cpu_ptr,
2286	dma_handle: tx->ring_dma_ptr);
2287	tx->ring_allocation_size = 0;
2288	tx->ring_cpu_ptr = NULL;
2289	tx->ring_dma_ptr = 0;
2290	}
2291	tx->ring_size = 0;
2292	}
2293
2294	static int lan743x_tx_ring_init(struct lan743x_tx *tx)
2295	{
2296	size_t ring_allocation_size = 0;
2297	void *cpu_ptr = NULL;
2298	dma_addr_t dma_ptr;
2299	int ret = -ENOMEM;
2300
2301	tx->ring_size = LAN743X_TX_RING_SIZE;
2302	if (tx->ring_size & ~TX_CFG_B_TX_RING_LEN_MASK_) {
2303	ret = -EINVAL;
2304	goto cleanup;
2305	}
2306	if (dma_set_mask_and_coherent(dev: &tx->adapter->pdev->dev,
2307	DMA_BIT_MASK(64))) {
2308	dev_warn(&tx->adapter->pdev->dev,
2309	"lan743x_: No suitable DMA available\n");
2310	ret = -ENOMEM;
2311	goto cleanup;
2312	}
2313	ring_allocation_size = ALIGN(tx->ring_size *
2314	sizeof(struct lan743x_tx_descriptor),
2315	PAGE_SIZE);
2316	dma_ptr = 0;
2317	cpu_ptr = dma_alloc_coherent(dev: &tx->adapter->pdev->dev,
2318	size: ring_allocation_size, dma_handle: &dma_ptr, GFP_KERNEL);
2319	if (!cpu_ptr) {
2320	ret = -ENOMEM;
2321	goto cleanup;
2322	}
2323
2324	tx->ring_allocation_size = ring_allocation_size;
2325	tx->ring_cpu_ptr = (struct lan743x_tx_descriptor *)cpu_ptr;
2326	tx->ring_dma_ptr = dma_ptr;
2327
2328	cpu_ptr = kcalloc(n: tx->ring_size, size: sizeof(*tx->buffer_info), GFP_KERNEL);
2329	if (!cpu_ptr) {
2330	ret = -ENOMEM;
2331	goto cleanup;
2332	}
2333	tx->buffer_info = (struct lan743x_tx_buffer_info *)cpu_ptr;
2334	dma_ptr = 0;
2335	cpu_ptr = dma_alloc_coherent(dev: &tx->adapter->pdev->dev,
2336	size: sizeof(*tx->head_cpu_ptr), dma_handle: &dma_ptr,
2337	GFP_KERNEL);
2338	if (!cpu_ptr) {
2339	ret = -ENOMEM;
2340	goto cleanup;
2341	}
2342
2343	tx->head_cpu_ptr = cpu_ptr;
2344	tx->head_dma_ptr = dma_ptr;
2345	if (tx->head_dma_ptr & 0x3) {
2346	ret = -ENOMEM;
2347	goto cleanup;
2348	}
2349
2350	return 0;
2351
2352	cleanup:
2353	lan743x_tx_ring_cleanup(tx);
2354	return ret;
2355	}
2356
2357	static void lan743x_tx_close(struct lan743x_tx *tx)
2358	{
2359	struct lan743x_adapter *adapter = tx->adapter;
2360
2361	lan743x_csr_write(adapter,
2362	DMAC_CMD,
2363	DMAC_CMD_STOP_T_(tx->channel_number));
2364	lan743x_dmac_tx_wait_till_stopped(adapter, tx_channel: tx->channel_number);
2365
2366	lan743x_csr_write(adapter,
2367	DMAC_INT_EN_CLR,
2368	DMAC_INT_BIT_TX_IOC_(tx->channel_number));
2369	lan743x_csr_write(adapter, INT_EN_CLR,
2370	INT_BIT_DMA_TX_(tx->channel_number));
2371	napi_disable(n: &tx->napi);
2372	netif_napi_del(napi: &tx->napi);
2373
2374	lan743x_csr_write(adapter, FCT_TX_CTL,
2375	FCT_TX_CTL_DIS_(tx->channel_number));
2376	lan743x_csr_wait_for_bit(adapter, FCT_TX_CTL,
2377	FCT_TX_CTL_EN_(tx->channel_number),
2378	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
2379
2380	lan743x_tx_release_all_descriptors(tx);
2381
2382	tx->rqd_descriptors = 0;
2383
2384	lan743x_tx_ring_cleanup(tx);
2385	}
2386
2387	static int lan743x_tx_open(struct lan743x_tx *tx)
2388	{
2389	struct lan743x_adapter *adapter = NULL;
2390	u32 data = 0;
2391	int ret;
2392
2393	adapter = tx->adapter;
2394	ret = lan743x_tx_ring_init(tx);
2395	if (ret)
2396	return ret;
2397
2398	/* initialize fifo */
2399	lan743x_csr_write(adapter, FCT_TX_CTL,
2400	FCT_TX_CTL_RESET_(tx->channel_number));
2401	lan743x_csr_wait_for_bit(adapter, FCT_TX_CTL,
2402	FCT_TX_CTL_RESET_(tx->channel_number),
2403	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
2404
2405	/* enable fifo */
2406	lan743x_csr_write(adapter, FCT_TX_CTL,
2407	FCT_TX_CTL_EN_(tx->channel_number));
2408
2409	/* reset tx channel */
2410	lan743x_csr_write(adapter, DMAC_CMD,
2411	DMAC_CMD_TX_SWR_(tx->channel_number));
2412	lan743x_csr_wait_for_bit(adapter, DMAC_CMD,
2413	DMAC_CMD_TX_SWR_(tx->channel_number),
2414	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
2415
2416	/* Write TX_BASE_ADDR */
2417	lan743x_csr_write(adapter,
2418	TX_BASE_ADDRH(tx->channel_number),
2419	DMA_ADDR_HIGH32(tx->ring_dma_ptr));
2420	lan743x_csr_write(adapter,
2421	TX_BASE_ADDRL(tx->channel_number),
2422	DMA_ADDR_LOW32(tx->ring_dma_ptr));
2423
2424	/* Write TX_CFG_B */
2425	data = lan743x_csr_read(adapter, TX_CFG_B(tx->channel_number));
2426	data &= ~TX_CFG_B_TX_RING_LEN_MASK_;
2427	data |= ((tx->ring_size) & TX_CFG_B_TX_RING_LEN_MASK_);
2428	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
2429	data |= TX_CFG_B_TDMABL_512_;
2430	lan743x_csr_write(adapter, TX_CFG_B(tx->channel_number), data);
2431
2432	/* Write TX_CFG_A */
2433	data = TX_CFG_A_TX_TMR_HPWB_SEL_IOC_ | TX_CFG_A_TX_HP_WB_EN_;
2434	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
2435	data |= TX_CFG_A_TX_HP_WB_ON_INT_TMR_;
2436	data |= TX_CFG_A_TX_PF_THRES_SET_(0x10);
2437	data |= TX_CFG_A_TX_PF_PRI_THRES_SET_(0x04);
2438	data |= TX_CFG_A_TX_HP_WB_THRES_SET_(0x07);
2439	}
2440	lan743x_csr_write(adapter, TX_CFG_A(tx->channel_number), data);
2441
2442	/* Write TX_HEAD_WRITEBACK_ADDR */
2443	lan743x_csr_write(adapter,
2444	TX_HEAD_WRITEBACK_ADDRH(tx->channel_number),
2445	DMA_ADDR_HIGH32(tx->head_dma_ptr));
2446	lan743x_csr_write(adapter,
2447	TX_HEAD_WRITEBACK_ADDRL(tx->channel_number),
2448	DMA_ADDR_LOW32(tx->head_dma_ptr));
2449
2450	/* set last head */
2451	tx->last_head = lan743x_csr_read(adapter, TX_HEAD(tx->channel_number));
2452
2453	/* write TX_TAIL */
2454	tx->last_tail = 0;
2455	lan743x_csr_write(adapter, TX_TAIL(tx->channel_number),
2456	data: (u32)(tx->last_tail));
2457	tx->vector_flags = lan743x_intr_get_vector_flags(adapter,
2458	INT_BIT_DMA_TX_
2459	(tx->channel_number));
2460	netif_napi_add_tx_weight(dev: adapter->netdev,
2461	napi: &tx->napi, poll: lan743x_tx_napi_poll,
2462	NAPI_POLL_WEIGHT);
2463	napi_enable(n: &tx->napi);
2464
2465	data = 0;
2466	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR)
2467	data |= TX_CFG_C_TX_TOP_INT_EN_AUTO_CLR_;
2468	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR)
2469	data |= TX_CFG_C_TX_DMA_INT_STS_AUTO_CLR_;
2470	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C)
2471	data |= TX_CFG_C_TX_INT_STS_R2C_MODE_MASK_;
2472	if (tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)
2473	data |= TX_CFG_C_TX_INT_EN_R2C_;
2474	lan743x_csr_write(adapter, TX_CFG_C(tx->channel_number), data);
2475
2476	if (!(tx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET))
2477	lan743x_csr_write(adapter, INT_EN_SET,
2478	INT_BIT_DMA_TX_(tx->channel_number));
2479	lan743x_csr_write(adapter, DMAC_INT_EN_SET,
2480	DMAC_INT_BIT_TX_IOC_(tx->channel_number));
2481
2482	/* start dmac channel */
2483	lan743x_csr_write(adapter, DMAC_CMD,
2484	DMAC_CMD_START_T_(tx->channel_number));
2485	return 0;
2486	}
2487
2488	static int lan743x_rx_next_index(struct lan743x_rx *rx, int index)
2489	{
2490	return ((++index) % rx->ring_size);
2491	}
2492
2493	static void lan743x_rx_update_tail(struct lan743x_rx *rx, int index)
2494	{
2495	/* update the tail once per 8 descriptors */
2496	if ((index & 7) == 7)
2497	lan743x_csr_write(adapter: rx->adapter, RX_TAIL(rx->channel_number),
2498	data: index);
2499	}
2500
2501	static int lan743x_rx_init_ring_element(struct lan743x_rx *rx, int index,
2502	gfp_t gfp)
2503	{
2504	struct net_device *netdev = rx->adapter->netdev;
2505	struct device *dev = &rx->adapter->pdev->dev;
2506	struct lan743x_rx_buffer_info *buffer_info;
2507	unsigned int buffer_length, used_length;
2508	struct lan743x_rx_descriptor *descriptor;
2509	struct sk_buff *skb;
2510	dma_addr_t dma_ptr;
2511
2512	buffer_length = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + RX_HEAD_PADDING;
2513
2514	descriptor = &rx->ring_cpu_ptr[index];
2515	buffer_info = &rx->buffer_info[index];
2516	skb = __netdev_alloc_skb(dev: netdev, length: buffer_length, gfp_mask: gfp);
2517	if (!skb)
2518	return -ENOMEM;
2519	dma_ptr = dma_map_single(dev, skb->data, buffer_length, DMA_FROM_DEVICE);
2520	if (dma_mapping_error(dev, dma_addr: dma_ptr)) {
2521	dev_kfree_skb_any(skb);
2522	return -ENOMEM;
2523	}
2524	if (buffer_info->dma_ptr) {
2525	/* sync used area of buffer only */
2526	if (le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_LS_)
2527	/* frame length is valid only if LS bit is set.
2528	* it's a safe upper bound for the used area in this
2529	* buffer.
2530	*/
2531	used_length = min(RX_DESC_DATA0_FRAME_LENGTH_GET_
2532	(le32_to_cpu(descriptor->data0)),
2533	buffer_info->buffer_length);
2534	else
2535	used_length = buffer_info->buffer_length;
2536	dma_sync_single_for_cpu(dev, addr: buffer_info->dma_ptr,
2537	size: used_length,
2538	dir: DMA_FROM_DEVICE);
2539	dma_unmap_single_attrs(dev, addr: buffer_info->dma_ptr,
2540	size: buffer_info->buffer_length,
2541	dir: DMA_FROM_DEVICE,
2542	DMA_ATTR_SKIP_CPU_SYNC);
2543	}
2544
2545	buffer_info->skb = skb;
2546	buffer_info->dma_ptr = dma_ptr;
2547	buffer_info->buffer_length = buffer_length;
2548	descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(buffer_info->dma_ptr));
2549	descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(buffer_info->dma_ptr));
2550	descriptor->data3 = 0;
2551	descriptor->data0 = cpu_to_le32((RX_DESC_DATA0_OWN_ |
2552	(buffer_length & RX_DESC_DATA0_BUF_LENGTH_MASK_)));
2553	lan743x_rx_update_tail(rx, index);
2554
2555	return 0;
2556	}
2557
2558	static void lan743x_rx_reuse_ring_element(struct lan743x_rx *rx, int index)
2559	{
2560	struct lan743x_rx_buffer_info *buffer_info;
2561	struct lan743x_rx_descriptor *descriptor;
2562
2563	descriptor = &rx->ring_cpu_ptr[index];
2564	buffer_info = &rx->buffer_info[index];
2565
2566	descriptor->data1 = cpu_to_le32(DMA_ADDR_LOW32(buffer_info->dma_ptr));
2567	descriptor->data2 = cpu_to_le32(DMA_ADDR_HIGH32(buffer_info->dma_ptr));
2568	descriptor->data3 = 0;
2569	descriptor->data0 = cpu_to_le32((RX_DESC_DATA0_OWN_ |
2570	((buffer_info->buffer_length) &
2571	RX_DESC_DATA0_BUF_LENGTH_MASK_)));
2572	lan743x_rx_update_tail(rx, index);
2573	}
2574
2575	static void lan743x_rx_release_ring_element(struct lan743x_rx *rx, int index)
2576	{
2577	struct lan743x_rx_buffer_info *buffer_info;
2578	struct lan743x_rx_descriptor *descriptor;
2579
2580	descriptor = &rx->ring_cpu_ptr[index];
2581	buffer_info = &rx->buffer_info[index];
2582
2583	memset(descriptor, 0, sizeof(*descriptor));
2584
2585	if (buffer_info->dma_ptr) {
2586	dma_unmap_single(&rx->adapter->pdev->dev,
2587	buffer_info->dma_ptr,
2588	buffer_info->buffer_length,
2589	DMA_FROM_DEVICE);
2590	buffer_info->dma_ptr = 0;
2591	}
2592
2593	if (buffer_info->skb) {
2594	dev_kfree_skb(buffer_info->skb);
2595	buffer_info->skb = NULL;
2596	}
2597
2598	memset(buffer_info, 0, sizeof(*buffer_info));
2599	}
2600
2601	static struct sk_buff *
2602	lan743x_rx_trim_skb(struct sk_buff *skb, int frame_length)
2603	{
2604	if (skb_linearize(skb)) {
2605	dev_kfree_skb_irq(skb);
2606	return NULL;
2607	}
2608	frame_length = max_t(int, 0, frame_length - ETH_FCS_LEN);
2609	if (skb->len > frame_length) {
2610	skb->tail -= skb->len - frame_length;
2611	skb->len = frame_length;
2612	}
2613	return skb;
2614	}
2615
2616	static int lan743x_rx_process_buffer(struct lan743x_rx *rx)
2617	{
2618	int current_head_index = le32_to_cpu(*rx->head_cpu_ptr);
2619	struct lan743x_rx_descriptor *descriptor, *desc_ext;
2620	struct net_device *netdev = rx->adapter->netdev;
2621	int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
2622	struct lan743x_rx_buffer_info *buffer_info;
2623	int frame_length, buffer_length;
2624	bool is_ice, is_tce, is_icsm;
2625	int extension_index = -1;
2626	bool is_last, is_first;
2627	struct sk_buff *skb;
2628
2629	if (current_head_index < 0 || current_head_index >= rx->ring_size)
2630	goto done;
2631
2632	if (rx->last_head < 0 || rx->last_head >= rx->ring_size)
2633	goto done;
2634
2635	if (rx->last_head == current_head_index)
2636	goto done;
2637
2638	descriptor = &rx->ring_cpu_ptr[rx->last_head];
2639	if (le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_OWN_)
2640	goto done;
2641	buffer_info = &rx->buffer_info[rx->last_head];
2642
2643	is_last = le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_LS_;
2644	is_first = le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_FS_;
2645
2646	if (is_last && le32_to_cpu(descriptor->data0) & RX_DESC_DATA0_EXT_) {
2647	/* extension is expected to follow */
2648	int index = lan743x_rx_next_index(rx, index: rx->last_head);
2649
2650	if (index == current_head_index)
2651	/* extension not yet available */
2652	goto done;
2653	desc_ext = &rx->ring_cpu_ptr[index];
2654	if (le32_to_cpu(desc_ext->data0) & RX_DESC_DATA0_OWN_)
2655	/* extension not yet available */
2656	goto done;
2657	if (!(le32_to_cpu(desc_ext->data0) & RX_DESC_DATA0_EXT_))
2658	goto move_forward;
2659	extension_index = index;
2660	}
2661
2662	/* Only the last buffer in a multi-buffer frame contains the total frame
2663	* length. The chip occasionally sends more buffers than strictly
2664	* required to reach the total frame length.
2665	* Handle this by adding all buffers to the skb in their entirety.
2666	* Once the real frame length is known, trim the skb.
2667	*/
2668	frame_length =
2669	RX_DESC_DATA0_FRAME_LENGTH_GET_(le32_to_cpu(descriptor->data0));
2670	buffer_length = buffer_info->buffer_length;
2671	is_ice = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICE_;
2672	is_tce = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_TCE_;
2673	is_icsm = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICSM_;
2674
2675	netdev_dbg(netdev, "%s%schunk: %d/%d",
2676	is_first ? "first " : " ",
2677	is_last ? "last " : " ",
2678	frame_length, buffer_length);
2679
2680	/* save existing skb, allocate new skb and map to dma */
2681	skb = buffer_info->skb;
2682	if (lan743x_rx_init_ring_element(rx, index: rx->last_head,
2683	GFP_ATOMIC | GFP_DMA)) {
2684	/* failed to allocate next skb.
2685	* Memory is very low.
2686	* Drop this packet and reuse buffer.
2687	*/
2688	lan743x_rx_reuse_ring_element(rx, index: rx->last_head);
2689	/* drop packet that was being assembled */
2690	dev_kfree_skb_irq(skb: rx->skb_head);
2691	rx->skb_head = NULL;
2692	goto process_extension;
2693	}
2694
2695	/* add buffers to skb via skb->frag_list */
2696	if (is_first) {
2697	skb_reserve(skb, RX_HEAD_PADDING);
2698	skb_put(skb, len: buffer_length - RX_HEAD_PADDING);
2699	if (rx->skb_head)
2700	dev_kfree_skb_irq(skb: rx->skb_head);
2701	rx->skb_head = skb;
2702	} else if (rx->skb_head) {
2703	skb_put(skb, len: buffer_length);
2704	if (skb_shinfo(rx->skb_head)->frag_list)
2705	rx->skb_tail->next = skb;
2706	else
2707	skb_shinfo(rx->skb_head)->frag_list = skb;
2708	rx->skb_tail = skb;
2709	rx->skb_head->len += skb->len;
2710	rx->skb_head->data_len += skb->len;
2711	rx->skb_head->truesize += skb->truesize;
2712	} else {
2713	/* packet to assemble has already been dropped because one or
2714	* more of its buffers could not be allocated
2715	*/
2716	netdev_dbg(netdev, "drop buffer intended for dropped packet");
2717	dev_kfree_skb_irq(skb);
2718	}
2719
2720	process_extension:
2721	if (extension_index >= 0) {
2722	u32 ts_sec;
2723	u32 ts_nsec;
2724
2725	ts_sec = le32_to_cpu(desc_ext->data1);
2726	ts_nsec = (le32_to_cpu(desc_ext->data2) &
2727	RX_DESC_DATA2_TS_NS_MASK_);
2728	if (rx->skb_head)
2729	skb_hwtstamps(skb: rx->skb_head)->hwtstamp =
2730	ktime_set(secs: ts_sec, nsecs: ts_nsec);
2731	lan743x_rx_reuse_ring_element(rx, index: extension_index);
2732	rx->last_head = extension_index;
2733	netdev_dbg(netdev, "process extension");
2734	}
2735
2736	if (is_last && rx->skb_head)
2737	rx->skb_head = lan743x_rx_trim_skb(skb: rx->skb_head, frame_length);
2738
2739	if (is_last && rx->skb_head) {
2740	rx->skb_head->protocol = eth_type_trans(skb: rx->skb_head,
2741	dev: rx->adapter->netdev);
2742	if (rx->adapter->netdev->features & NETIF_F_RXCSUM) {
2743	if (!is_ice && !is_tce && !is_icsm)
2744	skb->ip_summed = CHECKSUM_UNNECESSARY;
2745	}
2746	netdev_dbg(netdev, "sending %d byte frame to OS",
2747	rx->skb_head->len);
2748	napi_gro_receive(napi: &rx->napi, skb: rx->skb_head);
2749	rx->skb_head = NULL;
2750	}
2751
2752	move_forward:
2753	/* push tail and head forward */
2754	rx->last_tail = rx->last_head;
2755	rx->last_head = lan743x_rx_next_index(rx, index: rx->last_head);
2756	result = RX_PROCESS_RESULT_BUFFER_RECEIVED;
2757	done:
2758	return result;
2759	}
2760
2761	static int lan743x_rx_napi_poll(struct napi_struct *napi, int weight)
2762	{
2763	struct lan743x_rx *rx = container_of(napi, struct lan743x_rx, napi);
2764	struct lan743x_adapter *adapter = rx->adapter;
2765	int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
2766	u32 rx_tail_flags = 0;
2767	int count;
2768
2769	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_W2C) {
2770	/* clear int status bit before reading packet */
2771	lan743x_csr_write(adapter, DMAC_INT_STS,
2772	DMAC_INT_BIT_RXFRM_(rx->channel_number));
2773	}
2774	for (count = 0; count < weight; count++) {
2775	result = lan743x_rx_process_buffer(rx);
2776	if (result == RX_PROCESS_RESULT_NOTHING_TO_DO)
2777	break;
2778	}
2779	rx->frame_count += count;
2780	if (count == weight || result == RX_PROCESS_RESULT_BUFFER_RECEIVED)
2781	return weight;
2782
2783	if (!napi_complete_done(n: napi, work_done: count))
2784	return count;
2785
2786	/* re-arm interrupts, must write to rx tail on some chip variants */
2787	if (rx->vector_flags & LAN743X_VECTOR_FLAG_VECTOR_ENABLE_AUTO_SET)
2788	rx_tail_flags |= RX_TAIL_SET_TOP_INT_VEC_EN_;
2789	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_SET) {
2790	rx_tail_flags |= RX_TAIL_SET_TOP_INT_EN_;
2791	} else {
2792	lan743x_csr_write(adapter, INT_EN_SET,
2793	INT_BIT_DMA_RX_(rx->channel_number));
2794	}
2795
2796	if (rx_tail_flags)
2797	lan743x_csr_write(adapter, RX_TAIL(rx->channel_number),
2798	data: rx_tail_flags | rx->last_tail);
2799
2800	return count;
2801	}
2802
2803	static void lan743x_rx_ring_cleanup(struct lan743x_rx *rx)
2804	{
2805	if (rx->buffer_info && rx->ring_cpu_ptr) {
2806	int index;
2807
2808	for (index = 0; index < rx->ring_size; index++)
2809	lan743x_rx_release_ring_element(rx, index);
2810	}
2811
2812	if (rx->head_cpu_ptr) {
2813	dma_free_coherent(dev: &rx->adapter->pdev->dev,
2814	size: sizeof(*rx->head_cpu_ptr), cpu_addr: rx->head_cpu_ptr,
2815	dma_handle: rx->head_dma_ptr);
2816	rx->head_cpu_ptr = NULL;
2817	rx->head_dma_ptr = 0;
2818	}
2819
2820	kfree(objp: rx->buffer_info);
2821	rx->buffer_info = NULL;
2822
2823	if (rx->ring_cpu_ptr) {
2824	dma_free_coherent(dev: &rx->adapter->pdev->dev,
2825	size: rx->ring_allocation_size, cpu_addr: rx->ring_cpu_ptr,
2826	dma_handle: rx->ring_dma_ptr);
2827	rx->ring_allocation_size = 0;
2828	rx->ring_cpu_ptr = NULL;
2829	rx->ring_dma_ptr = 0;
2830	}
2831
2832	rx->ring_size = 0;
2833	rx->last_head = 0;
2834	}
2835
2836	static int lan743x_rx_ring_init(struct lan743x_rx *rx)
2837	{
2838	size_t ring_allocation_size = 0;
2839	dma_addr_t dma_ptr = 0;
2840	void *cpu_ptr = NULL;
2841	int ret = -ENOMEM;
2842	int index = 0;
2843
2844	rx->ring_size = LAN743X_RX_RING_SIZE;
2845	if (rx->ring_size <= 1) {
2846	ret = -EINVAL;
2847	goto cleanup;
2848	}
2849	if (rx->ring_size & ~RX_CFG_B_RX_RING_LEN_MASK_) {
2850	ret = -EINVAL;
2851	goto cleanup;
2852	}
2853	if (dma_set_mask_and_coherent(dev: &rx->adapter->pdev->dev,
2854	DMA_BIT_MASK(64))) {
2855	dev_warn(&rx->adapter->pdev->dev,
2856	"lan743x_: No suitable DMA available\n");
2857	ret = -ENOMEM;
2858	goto cleanup;
2859	}
2860	ring_allocation_size = ALIGN(rx->ring_size *
2861	sizeof(struct lan743x_rx_descriptor),
2862	PAGE_SIZE);
2863	dma_ptr = 0;
2864	cpu_ptr = dma_alloc_coherent(dev: &rx->adapter->pdev->dev,
2865	size: ring_allocation_size, dma_handle: &dma_ptr, GFP_KERNEL);
2866	if (!cpu_ptr) {
2867	ret = -ENOMEM;
2868	goto cleanup;
2869	}
2870	rx->ring_allocation_size = ring_allocation_size;
2871	rx->ring_cpu_ptr = (struct lan743x_rx_descriptor *)cpu_ptr;
2872	rx->ring_dma_ptr = dma_ptr;
2873
2874	cpu_ptr = kcalloc(n: rx->ring_size, size: sizeof(*rx->buffer_info),
2875	GFP_KERNEL);
2876	if (!cpu_ptr) {
2877	ret = -ENOMEM;
2878	goto cleanup;
2879	}
2880	rx->buffer_info = (struct lan743x_rx_buffer_info *)cpu_ptr;
2881	dma_ptr = 0;
2882	cpu_ptr = dma_alloc_coherent(dev: &rx->adapter->pdev->dev,
2883	size: sizeof(*rx->head_cpu_ptr), dma_handle: &dma_ptr,
2884	GFP_KERNEL);
2885	if (!cpu_ptr) {
2886	ret = -ENOMEM;
2887	goto cleanup;
2888	}
2889
2890	rx->head_cpu_ptr = cpu_ptr;
2891	rx->head_dma_ptr = dma_ptr;
2892	if (rx->head_dma_ptr & 0x3) {
2893	ret = -ENOMEM;
2894	goto cleanup;
2895	}
2896
2897	rx->last_head = 0;
2898	for (index = 0; index < rx->ring_size; index++) {
2899	ret = lan743x_rx_init_ring_element(rx, index, GFP_KERNEL);
2900	if (ret)
2901	goto cleanup;
2902	}
2903	return 0;
2904
2905	cleanup:
2906	netif_warn(rx->adapter, ifup, rx->adapter->netdev,
2907	"Error allocating memory for LAN743x\n");
2908
2909	lan743x_rx_ring_cleanup(rx);
2910	return ret;
2911	}
2912
2913	static void lan743x_rx_close(struct lan743x_rx *rx)
2914	{
2915	struct lan743x_adapter *adapter = rx->adapter;
2916
2917	lan743x_csr_write(adapter, FCT_RX_CTL,
2918	FCT_RX_CTL_DIS_(rx->channel_number));
2919	lan743x_csr_wait_for_bit(adapter, FCT_RX_CTL,
2920	FCT_RX_CTL_EN_(rx->channel_number),
2921	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
2922
2923	lan743x_csr_write(adapter, DMAC_CMD,
2924	DMAC_CMD_STOP_R_(rx->channel_number));
2925	lan743x_dmac_rx_wait_till_stopped(adapter, rx_channel: rx->channel_number);
2926
2927	lan743x_csr_write(adapter, DMAC_INT_EN_CLR,
2928	DMAC_INT_BIT_RXFRM_(rx->channel_number));
2929	lan743x_csr_write(adapter, INT_EN_CLR,
2930	INT_BIT_DMA_RX_(rx->channel_number));
2931	napi_disable(n: &rx->napi);
2932
2933	netif_napi_del(napi: &rx->napi);
2934
2935	lan743x_rx_ring_cleanup(rx);
2936	}
2937
2938	static int lan743x_rx_open(struct lan743x_rx *rx)
2939	{
2940	struct lan743x_adapter *adapter = rx->adapter;
2941	u32 data = 0;
2942	int ret;
2943
2944	rx->frame_count = 0;
2945	ret = lan743x_rx_ring_init(rx);
2946	if (ret)
2947	goto return_error;
2948
2949	netif_napi_add(dev: adapter->netdev, napi: &rx->napi, poll: lan743x_rx_napi_poll);
2950
2951	lan743x_csr_write(adapter, DMAC_CMD,
2952	DMAC_CMD_RX_SWR_(rx->channel_number));
2953	lan743x_csr_wait_for_bit(adapter, DMAC_CMD,
2954	DMAC_CMD_RX_SWR_(rx->channel_number),
2955	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
2956
2957	/* set ring base address */
2958	lan743x_csr_write(adapter,
2959	RX_BASE_ADDRH(rx->channel_number),
2960	DMA_ADDR_HIGH32(rx->ring_dma_ptr));
2961	lan743x_csr_write(adapter,
2962	RX_BASE_ADDRL(rx->channel_number),
2963	DMA_ADDR_LOW32(rx->ring_dma_ptr));
2964
2965	/* set rx write back address */
2966	lan743x_csr_write(adapter,
2967	RX_HEAD_WRITEBACK_ADDRH(rx->channel_number),
2968	DMA_ADDR_HIGH32(rx->head_dma_ptr));
2969	lan743x_csr_write(adapter,
2970	RX_HEAD_WRITEBACK_ADDRL(rx->channel_number),
2971	DMA_ADDR_LOW32(rx->head_dma_ptr));
2972	data = RX_CFG_A_RX_HP_WB_EN_;
2973	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0)) {
2974	data |= (RX_CFG_A_RX_WB_ON_INT_TMR_ |
2975	RX_CFG_A_RX_WB_THRES_SET_(0x7) |
2976	RX_CFG_A_RX_PF_THRES_SET_(16) |
2977	RX_CFG_A_RX_PF_PRI_THRES_SET_(4));
2978	}
2979
2980	/* set RX_CFG_A */
2981	lan743x_csr_write(adapter,
2982	RX_CFG_A(rx->channel_number), data);
2983
2984	/* set RX_CFG_B */
2985	data = lan743x_csr_read(adapter, RX_CFG_B(rx->channel_number));
2986	data &= ~RX_CFG_B_RX_PAD_MASK_;
2987	if (!RX_HEAD_PADDING)
2988	data |= RX_CFG_B_RX_PAD_0_;
2989	else
2990	data |= RX_CFG_B_RX_PAD_2_;
2991	data &= ~RX_CFG_B_RX_RING_LEN_MASK_;
2992	data |= ((rx->ring_size) & RX_CFG_B_RX_RING_LEN_MASK_);
2993	if (!(adapter->csr.flags & LAN743X_CSR_FLAG_IS_A0))
2994	data |= RX_CFG_B_RDMABL_512_;
2995
2996	lan743x_csr_write(adapter, RX_CFG_B(rx->channel_number), data);
2997	rx->vector_flags = lan743x_intr_get_vector_flags(adapter,
2998	INT_BIT_DMA_RX_
2999	(rx->channel_number));
3000
3001	/* set RX_CFG_C */
3002	data = 0;
3003	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_AUTO_CLEAR)
3004	data |= RX_CFG_C_RX_TOP_INT_EN_AUTO_CLR_;
3005	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_AUTO_CLEAR)
3006	data |= RX_CFG_C_RX_DMA_INT_STS_AUTO_CLR_;
3007	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_STATUS_R2C)
3008	data |= RX_CFG_C_RX_INT_STS_R2C_MODE_MASK_;
3009	if (rx->vector_flags & LAN743X_VECTOR_FLAG_SOURCE_ENABLE_R2C)
3010	data |= RX_CFG_C_RX_INT_EN_R2C_;
3011	lan743x_csr_write(adapter, RX_CFG_C(rx->channel_number), data);
3012
3013	rx->last_tail = ((u32)(rx->ring_size - 1));
3014	lan743x_csr_write(adapter, RX_TAIL(rx->channel_number),
3015	data: rx->last_tail);
3016	rx->last_head = lan743x_csr_read(adapter, RX_HEAD(rx->channel_number));
3017	if (rx->last_head) {
3018	ret = -EIO;
3019	goto napi_delete;
3020	}
3021
3022	napi_enable(n: &rx->napi);
3023
3024	lan743x_csr_write(adapter, INT_EN_SET,
3025	INT_BIT_DMA_RX_(rx->channel_number));
3026	lan743x_csr_write(adapter, DMAC_INT_STS,
3027	DMAC_INT_BIT_RXFRM_(rx->channel_number));
3028	lan743x_csr_write(adapter, DMAC_INT_EN_SET,
3029	DMAC_INT_BIT_RXFRM_(rx->channel_number));
3030	lan743x_csr_write(adapter, DMAC_CMD,
3031	DMAC_CMD_START_R_(rx->channel_number));
3032
3033	/* initialize fifo */
3034	lan743x_csr_write(adapter, FCT_RX_CTL,
3035	FCT_RX_CTL_RESET_(rx->channel_number));
3036	lan743x_csr_wait_for_bit(adapter, FCT_RX_CTL,
3037	FCT_RX_CTL_RESET_(rx->channel_number),
3038	target_value: 0, usleep_min: 1000, usleep_max: 20000, count: 100);
3039	lan743x_csr_write(adapter, FCT_FLOW(rx->channel_number),
3040	FCT_FLOW_CTL_REQ_EN_ |
3041	FCT_FLOW_CTL_ON_THRESHOLD_SET_(0x2A) |
3042	FCT_FLOW_CTL_OFF_THRESHOLD_SET_(0xA));
3043
3044	/* enable fifo */
3045	lan743x_csr_write(adapter, FCT_RX_CTL,
3046	FCT_RX_CTL_EN_(rx->channel_number));
3047	return 0;
3048
3049	napi_delete:
3050	netif_napi_del(napi: &rx->napi);
3051	lan743x_rx_ring_cleanup(rx);
3052
3053	return_error:
3054	return ret;
3055	}
3056
3057	static int lan743x_netdev_close(struct net_device *netdev)
3058	{
3059	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3060	int index;
3061
3062	for (index = 0; index < adapter->used_tx_channels; index++)
3063	lan743x_tx_close(tx: &adapter->tx[index]);
3064
3065	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++)
3066	lan743x_rx_close(rx: &adapter->rx[index]);
3067
3068	lan743x_ptp_close(adapter);
3069
3070	lan743x_phy_close(adapter);
3071
3072	lan743x_mac_close(adapter);
3073
3074	lan743x_intr_close(adapter);
3075
3076	return 0;
3077	}
3078
3079	static int lan743x_netdev_open(struct net_device *netdev)
3080	{
3081	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3082	int index;
3083	int ret;
3084
3085	ret = lan743x_intr_open(adapter);
3086	if (ret)
3087	goto return_error;
3088
3089	ret = lan743x_mac_open(adapter);
3090	if (ret)
3091	goto close_intr;
3092
3093	ret = lan743x_phy_open(adapter);
3094	if (ret)
3095	goto close_mac;
3096
3097	ret = lan743x_ptp_open(adapter);
3098	if (ret)
3099	goto close_phy;
3100
3101	lan743x_rfe_open(adapter);
3102
3103	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3104	ret = lan743x_rx_open(rx: &adapter->rx[index]);
3105	if (ret)
3106	goto close_rx;
3107	}
3108
3109	for (index = 0; index < adapter->used_tx_channels; index++) {
3110	ret = lan743x_tx_open(tx: &adapter->tx[index]);
3111	if (ret)
3112	goto close_tx;
3113	}
3114	return 0;
3115
3116	close_tx:
3117	for (index = 0; index < adapter->used_tx_channels; index++) {
3118	if (adapter->tx[index].ring_cpu_ptr)
3119	lan743x_tx_close(tx: &adapter->tx[index]);
3120	}
3121
3122	close_rx:
3123	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3124	if (adapter->rx[index].ring_cpu_ptr)
3125	lan743x_rx_close(rx: &adapter->rx[index]);
3126	}
3127	lan743x_ptp_close(adapter);
3128
3129	close_phy:
3130	lan743x_phy_close(adapter);
3131
3132	close_mac:
3133	lan743x_mac_close(adapter);
3134
3135	close_intr:
3136	lan743x_intr_close(adapter);
3137
3138	return_error:
3139	netif_warn(adapter, ifup, adapter->netdev,
3140	"Error opening LAN743x\n");
3141	return ret;
3142	}
3143
3144	static netdev_tx_t lan743x_netdev_xmit_frame(struct sk_buff *skb,
3145	struct net_device *netdev)
3146	{
3147	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3148	u8 ch = 0;
3149
3150	if (adapter->is_pci11x1x)
3151	ch = skb->queue_mapping % PCI11X1X_USED_TX_CHANNELS;
3152
3153	return lan743x_tx_xmit_frame(tx: &adapter->tx[ch], skb);
3154	}
3155
3156	static int lan743x_netdev_ioctl(struct net_device *netdev,
3157	struct ifreq *ifr, int cmd)
3158	{
3159	if (!netif_running(dev: netdev))
3160	return -EINVAL;
3161	if (cmd == SIOCSHWTSTAMP)
3162	return lan743x_ptp_ioctl(netdev, ifr, cmd);
3163	return phy_mii_ioctl(phydev: netdev->phydev, ifr, cmd);
3164	}
3165
3166	static void lan743x_netdev_set_multicast(struct net_device *netdev)
3167	{
3168	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3169
3170	lan743x_rfe_set_multicast(adapter);
3171	}
3172
3173	static int lan743x_netdev_change_mtu(struct net_device *netdev, int new_mtu)
3174	{
3175	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3176	int ret = 0;
3177
3178	ret = lan743x_mac_set_mtu(adapter, new_mtu);
3179	if (!ret)
3180	netdev->mtu = new_mtu;
3181	return ret;
3182	}
3183
3184	static void lan743x_netdev_get_stats64(struct net_device *netdev,
3185	struct rtnl_link_stats64 *stats)
3186	{
3187	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3188
3189	stats->rx_packets = lan743x_csr_read(adapter, STAT_RX_TOTAL_FRAMES);
3190	stats->tx_packets = lan743x_csr_read(adapter, STAT_TX_TOTAL_FRAMES);
3191	stats->rx_bytes = lan743x_csr_read(adapter,
3192	STAT_RX_UNICAST_BYTE_COUNT) +
3193	lan743x_csr_read(adapter,
3194	STAT_RX_BROADCAST_BYTE_COUNT) +
3195	lan743x_csr_read(adapter,
3196	STAT_RX_MULTICAST_BYTE_COUNT);
3197	stats->tx_bytes = lan743x_csr_read(adapter,
3198	STAT_TX_UNICAST_BYTE_COUNT) +
3199	lan743x_csr_read(adapter,
3200	STAT_TX_BROADCAST_BYTE_COUNT) +
3201	lan743x_csr_read(adapter,
3202	STAT_TX_MULTICAST_BYTE_COUNT);
3203	stats->rx_errors = lan743x_csr_read(adapter, STAT_RX_FCS_ERRORS) +
3204	lan743x_csr_read(adapter,
3205	STAT_RX_ALIGNMENT_ERRORS) +
3206	lan743x_csr_read(adapter, STAT_RX_JABBER_ERRORS) +
3207	lan743x_csr_read(adapter,
3208	STAT_RX_UNDERSIZE_FRAME_ERRORS) +
3209	lan743x_csr_read(adapter,
3210	STAT_RX_OVERSIZE_FRAME_ERRORS);
3211	stats->tx_errors = lan743x_csr_read(adapter, STAT_TX_FCS_ERRORS) +
3212	lan743x_csr_read(adapter,
3213	STAT_TX_EXCESS_DEFERRAL_ERRORS) +
3214	lan743x_csr_read(adapter, STAT_TX_CARRIER_ERRORS);
3215	stats->rx_dropped = lan743x_csr_read(adapter,
3216	STAT_RX_DROPPED_FRAMES);
3217	stats->tx_dropped = lan743x_csr_read(adapter,
3218	STAT_TX_EXCESSIVE_COLLISION);
3219	stats->multicast = lan743x_csr_read(adapter,
3220	STAT_RX_MULTICAST_FRAMES) +
3221	lan743x_csr_read(adapter,
3222	STAT_TX_MULTICAST_FRAMES);
3223	stats->collisions = lan743x_csr_read(adapter,
3224	STAT_TX_SINGLE_COLLISIONS) +
3225	lan743x_csr_read(adapter,
3226	STAT_TX_MULTIPLE_COLLISIONS) +
3227	lan743x_csr_read(adapter,
3228	STAT_TX_LATE_COLLISIONS);
3229	}
3230
3231	static int lan743x_netdev_set_mac_address(struct net_device *netdev,
3232	void *addr)
3233	{
3234	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3235	struct sockaddr *sock_addr = addr;
3236	int ret;
3237
3238	ret = eth_prepare_mac_addr_change(dev: netdev, p: sock_addr);
3239	if (ret)
3240	return ret;
3241	eth_hw_addr_set(dev: netdev, addr: sock_addr->sa_data);
3242	lan743x_mac_set_address(adapter, addr: sock_addr->sa_data);
3243	lan743x_rfe_update_mac_address(adapter);
3244	return 0;
3245	}
3246
3247	static const struct net_device_ops lan743x_netdev_ops = {
3248	.ndo_open = lan743x_netdev_open,
3249	.ndo_stop = lan743x_netdev_close,
3250	.ndo_start_xmit = lan743x_netdev_xmit_frame,
3251	.ndo_eth_ioctl = lan743x_netdev_ioctl,
3252	.ndo_set_rx_mode = lan743x_netdev_set_multicast,
3253	.ndo_change_mtu = lan743x_netdev_change_mtu,
3254	.ndo_get_stats64 = lan743x_netdev_get_stats64,
3255	.ndo_set_mac_address = lan743x_netdev_set_mac_address,
3256	};
3257
3258	static void lan743x_hardware_cleanup(struct lan743x_adapter *adapter)
3259	{
3260	lan743x_csr_write(adapter, INT_EN_CLR, data: 0xFFFFFFFF);
3261	}
3262
3263	static void lan743x_mdiobus_cleanup(struct lan743x_adapter *adapter)
3264	{
3265	mdiobus_unregister(bus: adapter->mdiobus);
3266	}
3267
3268	static void lan743x_full_cleanup(struct lan743x_adapter *adapter)
3269	{
3270	unregister_netdev(dev: adapter->netdev);
3271
3272	lan743x_mdiobus_cleanup(adapter);
3273	lan743x_hardware_cleanup(adapter);
3274	lan743x_pci_cleanup(adapter);
3275	}
3276
3277	static void pci11x1x_set_rfe_rd_fifo_threshold(struct lan743x_adapter *adapter)
3278	{
3279	u16 rev = adapter->csr.id_rev & ID_REV_CHIP_REV_MASK_;
3280
3281	if (rev == ID_REV_CHIP_REV_PCI11X1X_B0_) {
3282	u32 misc_ctl;
3283
3284	misc_ctl = lan743x_csr_read(adapter, MISC_CTL_0);
3285	misc_ctl &= ~MISC_CTL_0_RFE_READ_FIFO_MASK_;
3286	misc_ctl |= FIELD_PREP(MISC_CTL_0_RFE_READ_FIFO_MASK_,
3287	RFE_RD_FIFO_TH_3_DWORDS);
3288	lan743x_csr_write(adapter, MISC_CTL_0, data: misc_ctl);
3289	}
3290	}
3291
3292	static int lan743x_hardware_init(struct lan743x_adapter *adapter,
3293	struct pci_dev *pdev)
3294	{
3295	struct lan743x_tx *tx;
3296	int index;
3297	int ret;
3298
3299	adapter->is_pci11x1x = is_pci11x1x_chip(adapter);
3300	if (adapter->is_pci11x1x) {
3301	adapter->max_tx_channels = PCI11X1X_MAX_TX_CHANNELS;
3302	adapter->used_tx_channels = PCI11X1X_USED_TX_CHANNELS;
3303	adapter->max_vector_count = PCI11X1X_MAX_VECTOR_COUNT;
3304	pci11x1x_strap_get_status(adapter);
3305	spin_lock_init(&adapter->eth_syslock_spinlock);
3306	mutex_init(&adapter->sgmii_rw_lock);
3307	pci11x1x_set_rfe_rd_fifo_threshold(adapter);
3308	} else {
3309	adapter->max_tx_channels = LAN743X_MAX_TX_CHANNELS;
3310	adapter->used_tx_channels = LAN743X_USED_TX_CHANNELS;
3311	adapter->max_vector_count = LAN743X_MAX_VECTOR_COUNT;
3312	}
3313
3314	adapter->intr.irq = adapter->pdev->irq;
3315	lan743x_csr_write(adapter, INT_EN_CLR, data: 0xFFFFFFFF);
3316
3317	ret = lan743x_gpio_init(adapter);
3318	if (ret)
3319	return ret;
3320
3321	ret = lan743x_mac_init(adapter);
3322	if (ret)
3323	return ret;
3324
3325	ret = lan743x_phy_init(adapter);
3326	if (ret)
3327	return ret;
3328
3329	ret = lan743x_ptp_init(adapter);
3330	if (ret)
3331	return ret;
3332
3333	lan743x_rfe_update_mac_address(adapter);
3334
3335	ret = lan743x_dmac_init(adapter);
3336	if (ret)
3337	return ret;
3338
3339	for (index = 0; index < LAN743X_USED_RX_CHANNELS; index++) {
3340	adapter->rx[index].adapter = adapter;
3341	adapter->rx[index].channel_number = index;
3342	}
3343
3344	for (index = 0; index < adapter->used_tx_channels; index++) {
3345	tx = &adapter->tx[index];
3346	tx->adapter = adapter;
3347	tx->channel_number = index;
3348	spin_lock_init(&tx->ring_lock);
3349	}
3350
3351	return 0;
3352	}
3353
3354	static int lan743x_mdiobus_init(struct lan743x_adapter *adapter)
3355	{
3356	u32 sgmii_ctl;
3357	int ret;
3358
3359	adapter->mdiobus = devm_mdiobus_alloc(dev: &adapter->pdev->dev);
3360	if (!(adapter->mdiobus)) {
3361	ret = -ENOMEM;
3362	goto return_error;
3363	}
3364
3365	adapter->mdiobus->priv = (void *)adapter;
3366	if (adapter->is_pci11x1x) {
3367	if (adapter->is_sgmii_en) {
3368	sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL);
3369	sgmii_ctl |= SGMII_CTL_SGMII_ENABLE_;
3370	sgmii_ctl &= ~SGMII_CTL_SGMII_POWER_DN_;
3371	lan743x_csr_write(adapter, SGMII_CTL, data: sgmii_ctl);
3372	netif_dbg(adapter, drv, adapter->netdev,
3373	"SGMII operation\n");
3374	adapter->mdiobus->read = lan743x_mdiobus_read_c22;
3375	adapter->mdiobus->write = lan743x_mdiobus_write_c22;
3376	adapter->mdiobus->read_c45 = lan743x_mdiobus_read_c45;
3377	adapter->mdiobus->write_c45 = lan743x_mdiobus_write_c45;
3378	adapter->mdiobus->name = "lan743x-mdiobus-c45";
3379	netif_dbg(adapter, drv, adapter->netdev,
3380	"lan743x-mdiobus-c45\n");
3381	} else {
3382	sgmii_ctl = lan743x_csr_read(adapter, SGMII_CTL);
3383	sgmii_ctl &= ~SGMII_CTL_SGMII_ENABLE_;
3384	sgmii_ctl |= SGMII_CTL_SGMII_POWER_DN_;
3385	lan743x_csr_write(adapter, SGMII_CTL, data: sgmii_ctl);
3386	netif_dbg(adapter, drv, adapter->netdev,
3387	"RGMII operation\n");
3388	// Only C22 support when RGMII I/F
3389	adapter->mdiobus->read = lan743x_mdiobus_read_c22;
3390	adapter->mdiobus->write = lan743x_mdiobus_write_c22;
3391	adapter->mdiobus->name = "lan743x-mdiobus";
3392	netif_dbg(adapter, drv, adapter->netdev,
3393	"lan743x-mdiobus\n");
3394	}
3395	} else {
3396	adapter->mdiobus->read = lan743x_mdiobus_read_c22;
3397	adapter->mdiobus->write = lan743x_mdiobus_write_c22;
3398	adapter->mdiobus->name = "lan743x-mdiobus";
3399	netif_dbg(adapter, drv, adapter->netdev, "lan743x-mdiobus\n");
3400	}
3401
3402	snprintf(buf: adapter->mdiobus->id, MII_BUS_ID_SIZE,
3403	fmt: "pci-%s", pci_name(pdev: adapter->pdev));
3404
3405	if ((adapter->csr.id_rev & ID_REV_ID_MASK_) == ID_REV_ID_LAN7430_)
3406	/* LAN7430 uses internal phy at address 1 */
3407	adapter->mdiobus->phy_mask = ~(u32)BIT(1);
3408
3409	/* register mdiobus */
3410	ret = mdiobus_register(adapter->mdiobus);
3411	if (ret < 0)
3412	goto return_error;
3413	return 0;
3414
3415	return_error:
3416	return ret;
3417	}
3418
3419	/* lan743x_pcidev_probe - Device Initialization Routine
3420	* @pdev: PCI device information struct
3421	* @id: entry in lan743x_pci_tbl
3422	*
3423	* Returns 0 on success, negative on failure
3424	*
3425	* initializes an adapter identified by a pci_dev structure.
3426	* The OS initialization, configuring of the adapter private structure,
3427	* and a hardware reset occur.
3428	**/
3429	static int lan743x_pcidev_probe(struct pci_dev *pdev,
3430	const struct pci_device_id *id)
3431	{
3432	struct lan743x_adapter *adapter = NULL;
3433	struct net_device *netdev = NULL;
3434	int ret = -ENODEV;
3435
3436	if (id->device == PCI_DEVICE_ID_SMSC_A011 ||
3437	id->device == PCI_DEVICE_ID_SMSC_A041) {
3438	netdev = devm_alloc_etherdev_mqs(dev: &pdev->dev,
3439	sizeof_priv: sizeof(struct lan743x_adapter),
3440	PCI11X1X_USED_TX_CHANNELS,
3441	LAN743X_USED_RX_CHANNELS);
3442	} else {
3443	netdev = devm_alloc_etherdev_mqs(dev: &pdev->dev,
3444	sizeof_priv: sizeof(struct lan743x_adapter),
3445	LAN743X_USED_TX_CHANNELS,
3446	LAN743X_USED_RX_CHANNELS);
3447	}
3448
3449	if (!netdev)
3450	goto return_error;
3451
3452	SET_NETDEV_DEV(netdev, &pdev->dev);
3453	pci_set_drvdata(pdev, data: netdev);
3454	adapter = netdev_priv(dev: netdev);
3455	adapter->netdev = netdev;
3456	adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |
3457	NETIF_MSG_LINK | NETIF_MSG_IFUP |
3458	NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;
3459	netdev->max_mtu = LAN743X_MAX_FRAME_SIZE;
3460
3461	of_get_mac_address(np: pdev->dev.of_node, mac: adapter->mac_address);
3462
3463	ret = lan743x_pci_init(adapter, pdev);
3464	if (ret)
3465	goto return_error;
3466
3467	ret = lan743x_csr_init(adapter);
3468	if (ret)
3469	goto cleanup_pci;
3470
3471	ret = lan743x_hardware_init(adapter, pdev);
3472	if (ret)
3473	goto cleanup_pci;
3474
3475	ret = lan743x_mdiobus_init(adapter);
3476	if (ret)
3477	goto cleanup_hardware;
3478
3479	adapter->netdev->netdev_ops = &lan743x_netdev_ops;
3480	adapter->netdev->ethtool_ops = &lan743x_ethtool_ops;
3481	adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO |
3482	NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
3483	adapter->netdev->hw_features = adapter->netdev->features;
3484
3485	/* carrier off reporting is important to ethtool even BEFORE open */
3486	netif_carrier_off(dev: netdev);
3487
3488	ret = register_netdev(dev: adapter->netdev);
3489	if (ret < 0)
3490	goto cleanup_mdiobus;
3491	return 0;
3492
3493	cleanup_mdiobus:
3494	lan743x_mdiobus_cleanup(adapter);
3495
3496	cleanup_hardware:
3497	lan743x_hardware_cleanup(adapter);
3498
3499	cleanup_pci:
3500	lan743x_pci_cleanup(adapter);
3501
3502	return_error:
3503	pr_warn("Initialization failed\n");
3504	return ret;
3505	}
3506
3507	/**
3508	* lan743x_pcidev_remove - Device Removal Routine
3509	* @pdev: PCI device information struct
3510	*
3511	* this is called by the PCI subsystem to alert the driver
3512	* that it should release a PCI device. This could be caused by a
3513	* Hot-Plug event, or because the driver is going to be removed from
3514	* memory.
3515	**/
3516	static void lan743x_pcidev_remove(struct pci_dev *pdev)
3517	{
3518	struct net_device *netdev = pci_get_drvdata(pdev);
3519	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3520
3521	lan743x_full_cleanup(adapter);
3522	}
3523
3524	static void lan743x_pcidev_shutdown(struct pci_dev *pdev)
3525	{
3526	struct net_device *netdev = pci_get_drvdata(pdev);
3527	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3528
3529	rtnl_lock();
3530	netif_device_detach(dev: netdev);
3531
3532	/* close netdev when netdev is at running state.
3533	* For instance, it is true when system goes to sleep by pm-suspend
3534	* However, it is false when system goes to sleep by suspend GUI menu
3535	*/
3536	if (netif_running(dev: netdev))
3537	lan743x_netdev_close(netdev);
3538	rtnl_unlock();
3539
3540	#ifdef CONFIG_PM
3541	pci_save_state(dev: pdev);
3542	#endif
3543
3544	/* clean up lan743x portion */
3545	lan743x_hardware_cleanup(adapter);
3546	}
3547
3548	#ifdef CONFIG_PM_SLEEP
3549	static u16 lan743x_pm_wakeframe_crc16(const u8 *buf, int len)
3550	{
3551	return bitrev16(crc16(0xFFFF, buf, len));
3552	}
3553
3554	static void lan743x_pm_set_wol(struct lan743x_adapter *adapter)
3555	{
3556	const u8 ipv4_multicast[3] = { 0x01, 0x00, 0x5E };
3557	const u8 ipv6_multicast[3] = { 0x33, 0x33 };
3558	const u8 arp_type[2] = { 0x08, 0x06 };
3559	int mask_index;
3560	u32 sopass;
3561	u32 pmtctl;
3562	u32 wucsr;
3563	u32 macrx;
3564	u16 crc;
3565
3566	for (mask_index = 0; mask_index < MAC_NUM_OF_WUF_CFG; mask_index++)
3567	lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index), data: 0);
3568
3569	/* clear wake settings */
3570	pmtctl = lan743x_csr_read(adapter, PMT_CTL);
3571	pmtctl |= PMT_CTL_WUPS_MASK_;
3572	pmtctl &= ~(PMT_CTL_GPIO_WAKEUP_EN_ | PMT_CTL_EEE_WAKEUP_EN_ |
3573	PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_ |
3574	PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_ | PMT_CTL_ETH_PHY_WAKE_EN_);
3575
3576	macrx = lan743x_csr_read(adapter, MAC_RX);
3577
3578	wucsr = 0;
3579	mask_index = 0;
3580
3581	pmtctl |= PMT_CTL_ETH_PHY_D3_COLD_OVR_ | PMT_CTL_ETH_PHY_D3_OVR_;
3582
3583	if (adapter->wolopts & WAKE_PHY) {
3584	pmtctl |= PMT_CTL_ETH_PHY_EDPD_PLL_CTL_;
3585	pmtctl |= PMT_CTL_ETH_PHY_WAKE_EN_;
3586	}
3587	if (adapter->wolopts & WAKE_MAGIC) {
3588	wucsr |= MAC_WUCSR_MPEN_;
3589	macrx |= MAC_RX_RXEN_;
3590	pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3591	}
3592	if (adapter->wolopts & WAKE_UCAST) {
3593	wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_PFDA_EN_;
3594	macrx |= MAC_RX_RXEN_;
3595	pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3596	pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3597	}
3598	if (adapter->wolopts & WAKE_BCAST) {
3599	wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_BCST_EN_;
3600	macrx |= MAC_RX_RXEN_;
3601	pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3602	pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3603	}
3604	if (adapter->wolopts & WAKE_MCAST) {
3605	/* IPv4 multicast */
3606	crc = lan743x_pm_wakeframe_crc16(buf: ipv4_multicast, len: 3);
3607	lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3608	MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_MCAST_ |
3609	(0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3610	(crc & MAC_WUF_CFG_CRC16_MASK_));
3611	lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), data: 7);
3612	lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), data: 0);
3613	lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), data: 0);
3614	lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), data: 0);
3615	mask_index++;
3616
3617	/* IPv6 multicast */
3618	crc = lan743x_pm_wakeframe_crc16(buf: ipv6_multicast, len: 2);
3619	lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3620	MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_MCAST_ |
3621	(0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3622	(crc & MAC_WUF_CFG_CRC16_MASK_));
3623	lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), data: 3);
3624	lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), data: 0);
3625	lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), data: 0);
3626	lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), data: 0);
3627	mask_index++;
3628
3629	wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_WAKE_EN_;
3630	macrx |= MAC_RX_RXEN_;
3631	pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3632	pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3633	}
3634	if (adapter->wolopts & WAKE_ARP) {
3635	/* set MAC_WUF_CFG & WUF_MASK
3636	* for packettype (offset 12,13) = ARP (0x0806)
3637	*/
3638	crc = lan743x_pm_wakeframe_crc16(buf: arp_type, len: 2);
3639	lan743x_csr_write(adapter, MAC_WUF_CFG(mask_index),
3640	MAC_WUF_CFG_EN_ | MAC_WUF_CFG_TYPE_ALL_ |
3641	(0 << MAC_WUF_CFG_OFFSET_SHIFT_) |
3642	(crc & MAC_WUF_CFG_CRC16_MASK_));
3643	lan743x_csr_write(adapter, MAC_WUF_MASK0(mask_index), data: 0x3000);
3644	lan743x_csr_write(adapter, MAC_WUF_MASK1(mask_index), data: 0);
3645	lan743x_csr_write(adapter, MAC_WUF_MASK2(mask_index), data: 0);
3646	lan743x_csr_write(adapter, MAC_WUF_MASK3(mask_index), data: 0);
3647	mask_index++;
3648
3649	wucsr |= MAC_WUCSR_RFE_WAKE_EN_ | MAC_WUCSR_WAKE_EN_;
3650	macrx |= MAC_RX_RXEN_;
3651	pmtctl |= PMT_CTL_WOL_EN_ | PMT_CTL_MAC_D3_RX_CLK_OVR_;
3652	pmtctl |= PMT_CTL_RX_FCT_RFE_D3_CLK_OVR_;
3653	}
3654
3655	if (adapter->wolopts & WAKE_MAGICSECURE) {
3656	sopass = *(u32 *)adapter->sopass;
3657	lan743x_csr_write(adapter, MAC_MP_SO_LO, data: sopass);
3658	sopass = *(u16 *)&adapter->sopass[4];
3659	lan743x_csr_write(adapter, MAC_MP_SO_HI, data: sopass);
3660	wucsr |= MAC_MP_SO_EN_;
3661	}
3662
3663	lan743x_csr_write(adapter, MAC_WUCSR, data: wucsr);
3664	lan743x_csr_write(adapter, PMT_CTL, data: pmtctl);
3665	lan743x_csr_write(adapter, MAC_RX, data: macrx);
3666	}
3667
3668	static int lan743x_pm_suspend(struct device *dev)
3669	{
3670	struct pci_dev *pdev = to_pci_dev(dev);
3671	struct net_device *netdev = pci_get_drvdata(pdev);
3672	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3673	u32 data;
3674
3675	lan743x_pcidev_shutdown(pdev);
3676
3677	/* clear all wakes */
3678	lan743x_csr_write(adapter, MAC_WUCSR, data: 0);
3679	lan743x_csr_write(adapter, MAC_WUCSR2, data: 0);
3680	lan743x_csr_write(adapter, MAC_WK_SRC, data: 0xFFFFFFFF);
3681
3682	if (adapter->wolopts)
3683	lan743x_pm_set_wol(adapter);
3684
3685	if (adapter->is_pci11x1x) {
3686	/* Save HW_CFG to config again in PM resume */
3687	data = lan743x_csr_read(adapter, HW_CFG);
3688	adapter->hw_cfg = data;
3689	data |= (HW_CFG_RST_PROTECT_PCIE_ |
3690	HW_CFG_D3_RESET_DIS_ |
3691	HW_CFG_D3_VAUX_OVR_ |
3692	HW_CFG_HOT_RESET_DIS_ |
3693	HW_CFG_RST_PROTECT_);
3694	lan743x_csr_write(adapter, HW_CFG, data);
3695	}
3696
3697	/* Host sets PME_En, put D3hot */
3698	return pci_prepare_to_sleep(dev: pdev);
3699	}
3700
3701	static int lan743x_pm_resume(struct device *dev)
3702	{
3703	struct pci_dev *pdev = to_pci_dev(dev);
3704	struct net_device *netdev = pci_get_drvdata(pdev);
3705	struct lan743x_adapter *adapter = netdev_priv(dev: netdev);
3706	int ret;
3707
3708	pci_set_power_state(dev: pdev, PCI_D0);
3709	pci_restore_state(dev: pdev);
3710	pci_save_state(dev: pdev);
3711
3712	/* Restore HW_CFG that was saved during pm suspend */
3713	if (adapter->is_pci11x1x)
3714	lan743x_csr_write(adapter, HW_CFG, data: adapter->hw_cfg);
3715
3716	ret = lan743x_hardware_init(adapter, pdev);
3717	if (ret) {
3718	netif_err(adapter, probe, adapter->netdev,
3719	"lan743x_hardware_init returned %d\n", ret);
3720	lan743x_pci_cleanup(adapter);
3721	return ret;
3722	}
3723
3724	/* open netdev when netdev is at running state while resume.
3725	* For instance, it is true when system wakesup after pm-suspend
3726	* However, it is false when system wakes up after suspend GUI menu
3727	*/
3728	if (netif_running(dev: netdev))
3729	lan743x_netdev_open(netdev);
3730
3731	netif_device_attach(dev: netdev);
3732	ret = lan743x_csr_read(adapter, MAC_WK_SRC);
3733	netif_info(adapter, drv, adapter->netdev,
3734	"Wakeup source : 0x%08X\n", ret);
3735
3736	return 0;
3737	}
3738
3739	static const struct dev_pm_ops lan743x_pm_ops = {
3740	SET_SYSTEM_SLEEP_PM_OPS(lan743x_pm_suspend, lan743x_pm_resume)
3741	};
3742	#endif /* CONFIG_PM_SLEEP */
3743
3744	static const struct pci_device_id lan743x_pcidev_tbl[] = {
3745	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_LAN7430) },
3746	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_LAN7431) },
3747	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_A011) },
3748	{ PCI_DEVICE(PCI_VENDOR_ID_SMSC, PCI_DEVICE_ID_SMSC_A041) },
3749	{ 0, }
3750	};
3751
3752	MODULE_DEVICE_TABLE(pci, lan743x_pcidev_tbl);
3753
3754	static struct pci_driver lan743x_pcidev_driver = {
3755	.name = DRIVER_NAME,
3756	.id_table = lan743x_pcidev_tbl,
3757	.probe = lan743x_pcidev_probe,
3758	.remove = lan743x_pcidev_remove,
3759	#ifdef CONFIG_PM_SLEEP
3760	.driver.pm = &lan743x_pm_ops,
3761	#endif
3762	.shutdown = lan743x_pcidev_shutdown,
3763	};
3764
3765	module_pci_driver(lan743x_pcidev_driver);
3766
3767	MODULE_AUTHOR(DRIVER_AUTHOR);
3768	MODULE_DESCRIPTION(DRIVER_DESC);
3769	MODULE_LICENSE("GPL");
3770