1	// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2	/* Copyright(c) 2020 Realtek Corporation
3	*/
4
5	#include <linux/pci.h>
6
7	#include "mac.h"
8	#include "pci.h"
9	#include "reg.h"
10	#include "ser.h"
11
12	static bool rtw89_pci_disable_clkreq;
13	static bool rtw89_pci_disable_aspm_l1;
14	static bool rtw89_pci_disable_l1ss;
15	module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
16	module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
17	module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
18	MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
19	MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
20	MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
21
22	static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev,
23	u32 *phy_offset)
24	{
25	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
26	struct pci_dev *pdev = rtwpci->pdev;
27	u32 val;
28	int ret;
29
30	ret = pci_read_config_dword(dev: pdev, RTW89_PCIE_L1_STS_V1, val: &val);
31	if (ret)
32	return ret;
33
34	val = u32_get_bits(v: val, RTW89_BCFG_LINK_SPEED_MASK);
35	if (val == RTW89_PCIE_GEN1_SPEED) {
36	*phy_offset = R_RAC_DIRECT_OFFSET_G1;
37	} else if (val == RTW89_PCIE_GEN2_SPEED) {
38	*phy_offset = R_RAC_DIRECT_OFFSET_G2;
39	} else {
40	rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val);
41	return -EFAULT;
42	}
43
44	return 0;
45	}
46
47	static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
48	{
49	u32 val;
50	int ret;
51
52	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
53
54	ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
55	1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
56	rtwdev, R_AX_PCIE_INIT_CFG1);
57
58	return ret;
59	}
60
61	static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
62	struct rtw89_pci_dma_ring *bd_ring,
63	u32 cur_idx, bool tx)
64	{
65	const struct rtw89_pci_info *info = rtwdev->pci_info;
66	u32 cnt, cur_rp, wp, rp, len;
67
68	rp = bd_ring->rp;
69	wp = bd_ring->wp;
70	len = bd_ring->len;
71
72	cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
73	if (tx) {
74	cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
75	} else {
76	if (info->rx_ring_eq_is_full)
77	wp += 1;
78
79	cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
80	}
81
82	bd_ring->rp = cur_rp;
83
84	return cnt;
85	}
86
87	static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
88	struct rtw89_pci_tx_ring *tx_ring)
89	{
90	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
91	u32 addr_idx = bd_ring->addr.idx;
92	u32 cnt, idx;
93
94	idx = rtw89_read32(rtwdev, addr: addr_idx);
95	cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, cur_idx: idx, tx: true);
96
97	return cnt;
98	}
99
100	static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
101	struct rtw89_pci *rtwpci,
102	u32 cnt, bool release_all)
103	{
104	struct rtw89_pci_tx_data *tx_data;
105	struct sk_buff *skb;
106	u32 qlen;
107
108	while (cnt--) {
109	skb = skb_dequeue(list: &rtwpci->h2c_queue);
110	if (!skb) {
111	rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
112	return;
113	}
114	skb_queue_tail(list: &rtwpci->h2c_release_queue, newsk: skb);
115	}
116
117	qlen = skb_queue_len(list_: &rtwpci->h2c_release_queue);
118	if (!release_all)
119	qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
120
121	while (qlen--) {
122	skb = skb_dequeue(list: &rtwpci->h2c_release_queue);
123	if (!skb) {
124	rtw89_err(rtwdev, "failed to release fwcmd\n");
125	return;
126	}
127	tx_data = RTW89_PCI_TX_SKB_CB(skb);
128	dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
129	DMA_TO_DEVICE);
130	dev_kfree_skb_any(skb);
131	}
132	}
133
134	static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
135	struct rtw89_pci *rtwpci)
136	{
137	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
138	u32 cnt;
139
140	cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
141	if (!cnt)
142	return;
143	rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, release_all: false);
144	}
145
146	static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
147	struct rtw89_pci_rx_ring *rx_ring)
148	{
149	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
150	u32 addr_idx = bd_ring->addr.idx;
151	u32 cnt, idx;
152
153	idx = rtw89_read32(rtwdev, addr: addr_idx);
154	cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, cur_idx: idx, tx: false);
155
156	return cnt;
157	}
158
159	static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
160	struct sk_buff *skb)
161	{
162	struct rtw89_pci_rx_info *rx_info;
163	dma_addr_t dma;
164
165	rx_info = RTW89_PCI_RX_SKB_CB(skb);
166	dma = rx_info->dma;
167	dma_sync_single_for_cpu(dev: rtwdev->dev, addr: dma, RTW89_PCI_RX_BUF_SIZE,
168	dir: DMA_FROM_DEVICE);
169	}
170
171	static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
172	struct sk_buff *skb)
173	{
174	struct rtw89_pci_rx_info *rx_info;
175	dma_addr_t dma;
176
177	rx_info = RTW89_PCI_RX_SKB_CB(skb);
178	dma = rx_info->dma;
179	dma_sync_single_for_device(dev: rtwdev->dev, addr: dma, RTW89_PCI_RX_BUF_SIZE,
180	dir: DMA_FROM_DEVICE);
181	}
182
183	static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
184	struct sk_buff *skb)
185	{
186	struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
187	struct rtw89_pci_rxbd_info *rxbd_info;
188	__le32 info;
189
190	rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
191	info = rxbd_info->dword;
192
193	rx_info->fs = le32_get_bits(v: info, RTW89_PCI_RXBD_FS);
194	rx_info->ls = le32_get_bits(v: info, RTW89_PCI_RXBD_LS);
195	rx_info->len = le32_get_bits(v: info, RTW89_PCI_RXBD_WRITE_SIZE);
196	rx_info->tag = le32_get_bits(v: info, RTW89_PCI_RXBD_TAG);
197	}
198
199	static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev,
200	struct rtw89_pci_rx_ring *rx_ring,
201	struct sk_buff *skb)
202	{
203	struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
204	const struct rtw89_pci_info *info = rtwdev->pci_info;
205	u32 target_rx_tag;
206
207	if (!info->check_rx_tag)
208	return 0;
209
210	/* valid range is 1 ~ 0x1FFF */
211	if (rx_ring->target_rx_tag == 0)
212	target_rx_tag = 1;
213	else
214	target_rx_tag = rx_ring->target_rx_tag;
215
216	if (rx_info->tag != target_rx_tag) {
217	rtw89_debug(rtwdev, mask: RTW89_DBG_UNEXP, fmt: "mismatch RX tag 0x%x 0x%x\n",
218	rx_info->tag, target_rx_tag);
219	return -EAGAIN;
220	}
221
222	return 0;
223	}
224
225	static
226	int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev,
227	struct rtw89_pci_rx_ring *rx_ring,
228	struct sk_buff *skb)
229	{
230	struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
231	int rx_tag_retry = 1000;
232	int ret;
233
234	do {
235	rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
236	rtw89_pci_rxbd_info_update(rtwdev, skb);
237
238	ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb);
239	if (ret != -EAGAIN)
240	break;
241	} while (rx_tag_retry--);
242
243	/* update target rx_tag for next RX */
244	rx_ring->target_rx_tag = rx_info->tag + 1;
245
246	return ret;
247	}
248
249	static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable)
250	{
251	const struct rtw89_pci_info *info = rtwdev->pci_info;
252	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
253	const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
254
255	if (enable) {
256	rtw89_write32_clr(rtwdev, addr: dma_stop1->addr, bit: dma_stop1->mask);
257	if (dma_stop2->addr)
258	rtw89_write32_clr(rtwdev, addr: dma_stop2->addr, bit: dma_stop2->mask);
259	} else {
260	rtw89_write32_set(rtwdev, addr: dma_stop1->addr, bit: dma_stop1->mask);
261	if (dma_stop2->addr)
262	rtw89_write32_set(rtwdev, addr: dma_stop2->addr, bit: dma_stop2->mask);
263	}
264	}
265
266	static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable)
267	{
268	const struct rtw89_pci_info *info = rtwdev->pci_info;
269	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
270
271	if (enable)
272	rtw89_write32_clr(rtwdev, addr: dma_stop1->addr, B_AX_STOP_CH12);
273	else
274	rtw89_write32_set(rtwdev, addr: dma_stop1->addr, B_AX_STOP_CH12);
275	}
276
277	static bool
278	rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
279	struct sk_buff *new,
280	const struct sk_buff *skb, u32 offset,
281	const struct rtw89_pci_rx_info *rx_info,
282	const struct rtw89_rx_desc_info *desc_info)
283	{
284	u32 copy_len = rx_info->len - offset;
285
286	if (unlikely(skb_tailroom(new) < copy_len)) {
287	rtw89_debug(rtwdev, mask: RTW89_DBG_TXRX,
288	fmt: "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
289	rx_info->len, desc_info->pkt_size, offset, fs, ls);
290	rtw89_hex_dump(rtwdev, mask: RTW89_DBG_TXRX, prefix_str: "rx_data: ",
291	buf: skb->data, len: rx_info->len);
292	/* length of a single segment skb is desc_info->pkt_size */
293	if (fs && ls) {
294	copy_len = desc_info->pkt_size;
295	} else {
296	rtw89_info(rtwdev, "drop rx data due to invalid length\n");
297	return false;
298	}
299	}
300
301	skb_put_data(skb: new, data: skb->data + offset, len: copy_len);
302
303	return true;
304	}
305
306	static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
307	struct rtw89_pci_dma_ring *bd_ring)
308	{
309	const struct rtw89_pci_info *info = rtwdev->pci_info;
310	u32 wp = bd_ring->wp;
311
312	if (!info->rx_ring_eq_is_full)
313	return wp;
314
315	if (++wp >= bd_ring->len)
316	wp = 0;
317
318	return wp;
319	}
320
321	static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
322	struct rtw89_pci_rx_ring *rx_ring)
323	{
324	struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
325	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
326	const struct rtw89_pci_info *info = rtwdev->pci_info;
327	struct sk_buff *new = rx_ring->diliver_skb;
328	struct rtw89_pci_rx_info *rx_info;
329	struct sk_buff *skb;
330	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
331	u32 skb_idx;
332	u32 offset;
333	u32 cnt = 1;
334	bool fs, ls;
335	int ret;
336
337	skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
338	skb = rx_ring->buf[skb_idx];
339
340	ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
341	if (ret) {
342	rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
343	bd_ring->wp, ret);
344	goto err_sync_device;
345	}
346
347	rx_info = RTW89_PCI_RX_SKB_CB(skb);
348	fs = info->no_rxbd_fs ? !new : rx_info->fs;
349	ls = rx_info->ls;
350
351	if (unlikely(!fs || !ls))
352	rtw89_debug(rtwdev, mask: RTW89_DBG_UNEXP,
353	fmt: "unexpected fs/ls=%d/%d tag=%u len=%u new->len=%u\n",
354	fs, ls, rx_info->tag, rx_info->len, new ? new->len : 0);
355
356	if (fs) {
357	if (new) {
358	rtw89_debug(rtwdev, mask: RTW89_DBG_UNEXP,
359	fmt: "skb should not be ready before first segment start\n");
360	goto err_sync_device;
361	}
362	if (desc_info->ready) {
363	rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
364	goto err_sync_device;
365	}
366
367	rtw89_chip_query_rxdesc(rtwdev, desc_info, data: skb->data, data_offset: rxinfo_size);
368
369	new = rtw89_alloc_skb_for_rx(rtwdev, length: desc_info->pkt_size);
370	if (!new)
371	goto err_sync_device;
372
373	rx_ring->diliver_skb = new;
374
375	/* first segment has RX desc */
376	offset = desc_info->offset + desc_info->rxd_len;
377	} else {
378	offset = sizeof(struct rtw89_pci_rxbd_info);
379	if (!new) {
380	rtw89_debug(rtwdev, mask: RTW89_DBG_UNEXP, fmt: "no last skb\n");
381	goto err_sync_device;
382	}
383	}
384	if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
385	goto err_sync_device;
386	rtw89_pci_sync_skb_for_device(rtwdev, skb);
387	rtw89_pci_rxbd_increase(rx_ring, cnt: 1);
388
389	if (!desc_info->ready) {
390	rtw89_warn(rtwdev, "no rx desc information\n");
391	goto err_free_resource;
392	}
393	if (ls) {
394	rtw89_core_rx(rtwdev, desc_info, skb: new);
395	rx_ring->diliver_skb = NULL;
396	desc_info->ready = false;
397	}
398
399	return cnt;
400
401	err_sync_device:
402	rtw89_pci_sync_skb_for_device(rtwdev, skb);
403	rtw89_pci_rxbd_increase(rx_ring, cnt: 1);
404	err_free_resource:
405	if (new)
406	dev_kfree_skb_any(skb: new);
407	rx_ring->diliver_skb = NULL;
408	desc_info->ready = false;
409
410	return cnt;
411	}
412
413	static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
414	struct rtw89_pci_rx_ring *rx_ring,
415	u32 cnt)
416	{
417	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
418	u32 rx_cnt;
419
420	while (cnt && rtwdev->napi_budget_countdown > 0) {
421	rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
422	if (!rx_cnt) {
423	rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
424
425	/* skip the rest RXBD bufs */
426	rtw89_pci_rxbd_increase(rx_ring, cnt);
427	break;
428	}
429
430	cnt -= rx_cnt;
431	}
432
433	rtw89_write16(rtwdev, addr: bd_ring->addr.idx, data: bd_ring->wp);
434	}
435
436	static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
437	struct rtw89_pci *rtwpci, int budget)
438	{
439	struct rtw89_pci_rx_ring *rx_ring;
440	int countdown = rtwdev->napi_budget_countdown;
441	u32 cnt;
442
443	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
444
445	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
446	if (!cnt)
447	return 0;
448
449	cnt = min_t(u32, budget, cnt);
450
451	rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
452
453	/* In case of flushing pending SKBs, the countdown may exceed. */
454	if (rtwdev->napi_budget_countdown <= 0)
455	return budget;
456
457	return budget - countdown;
458	}
459
460	static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
461	struct rtw89_pci_tx_ring *tx_ring,
462	struct sk_buff *skb, u8 tx_status)
463	{
464	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
465	struct ieee80211_tx_info *info;
466
467	rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_done: tx_status == RTW89_TX_DONE);
468
469	info = IEEE80211_SKB_CB(skb);
470	ieee80211_tx_info_clear_status(info);
471
472	if (info->flags & IEEE80211_TX_CTL_NO_ACK)
473	info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
474	if (tx_status == RTW89_TX_DONE) {
475	info->flags |= IEEE80211_TX_STAT_ACK;
476	tx_ring->tx_acked++;
477	} else {
478	if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
479	rtw89_debug(rtwdev, mask: RTW89_DBG_FW,
480	fmt: "failed to TX of status %x\n", tx_status);
481	switch (tx_status) {
482	case RTW89_TX_RETRY_LIMIT:
483	tx_ring->tx_retry_lmt++;
484	break;
485	case RTW89_TX_LIFE_TIME:
486	tx_ring->tx_life_time++;
487	break;
488	case RTW89_TX_MACID_DROP:
489	tx_ring->tx_mac_id_drop++;
490	break;
491	default:
492	rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
493	break;
494	}
495	}
496
497	ieee80211_tx_status_ni(hw: rtwdev->hw, skb);
498	}
499
500	static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
501	{
502	struct rtw89_pci_tx_wd *txwd;
503	u32 cnt;
504
505	cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
506	while (cnt--) {
507	txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
508	if (!txwd) {
509	rtw89_warn(rtwdev, "No busy txwd pages available\n");
510	break;
511	}
512
513	list_del_init(entry: &txwd->list);
514
515	/* this skb has been freed by RPP */
516	if (skb_queue_len(list_: &txwd->queue) == 0)
517	rtw89_pci_enqueue_txwd(tx_ring, txwd);
518	}
519	}
520
521	static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
522	struct rtw89_pci_tx_ring *tx_ring)
523	{
524	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
525	struct rtw89_pci_tx_wd *txwd;
526	int i;
527
528	for (i = 0; i < wd_ring->page_num; i++) {
529	txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
530	if (!txwd)
531	break;
532
533	list_del_init(entry: &txwd->list);
534	}
535	}
536
537	static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
538	struct rtw89_pci_tx_ring *tx_ring,
539	struct rtw89_pci_tx_wd *txwd, u16 seq,
540	u8 tx_status)
541	{
542	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
543	struct rtw89_pci_tx_data *tx_data;
544	struct sk_buff *skb, *tmp;
545	u8 txch = tx_ring->txch;
546
547	if (!list_empty(head: &txwd->list)) {
548	rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
549	/* In low power mode, RPP can receive before updating of TX BD.
550	* In normal mode, it should not happen so give it a warning.
551	*/
552	if (!rtwpci->low_power && !list_empty(head: &txwd->list))
553	rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
554	txch, seq);
555	}
556
557	skb_queue_walk_safe(&txwd->queue, skb, tmp) {
558	skb_unlink(skb, list: &txwd->queue);
559
560	tx_data = RTW89_PCI_TX_SKB_CB(skb);
561	dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
562	DMA_TO_DEVICE);
563
564	rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
565	}
566
567	if (list_empty(head: &txwd->list))
568	rtw89_pci_enqueue_txwd(tx_ring, txwd);
569	}
570
571	static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
572	struct rtw89_pci_rpp_fmt *rpp)
573	{
574	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
575	struct rtw89_pci_tx_ring *tx_ring;
576	struct rtw89_pci_tx_wd_ring *wd_ring;
577	struct rtw89_pci_tx_wd *txwd;
578	u16 seq;
579	u8 qsel, tx_status, txch;
580
581	seq = le32_get_bits(v: rpp->dword, RTW89_PCI_RPP_SEQ);
582	qsel = le32_get_bits(v: rpp->dword, RTW89_PCI_RPP_QSEL);
583	tx_status = le32_get_bits(v: rpp->dword, RTW89_PCI_RPP_TX_STATUS);
584	txch = rtw89_core_get_ch_dma(rtwdev, qsel);
585
586	if (txch == RTW89_TXCH_CH12) {
587	rtw89_warn(rtwdev, "should no fwcmd release report\n");
588	return;
589	}
590
591	tx_ring = &rtwpci->tx_rings[txch];
592	wd_ring = &tx_ring->wd_ring;
593	txwd = &wd_ring->pages[seq];
594
595	rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
596	}
597
598	static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
599	struct rtw89_pci_tx_ring *tx_ring)
600	{
601	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
602	struct rtw89_pci_tx_wd *txwd;
603	int i;
604
605	for (i = 0; i < wd_ring->page_num; i++) {
606	txwd = &wd_ring->pages[i];
607
608	if (!list_empty(head: &txwd->list))
609	continue;
610
611	rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq: i, RTW89_TX_MACID_DROP);
612	}
613	}
614
615	static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
616	struct rtw89_pci_rx_ring *rx_ring,
617	u32 max_cnt)
618	{
619	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
620	struct rtw89_pci_rx_info *rx_info;
621	struct rtw89_pci_rpp_fmt *rpp;
622	struct rtw89_rx_desc_info desc_info = {};
623	struct sk_buff *skb;
624	u32 cnt = 0;
625	u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
626	u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
627	u32 skb_idx;
628	u32 offset;
629	int ret;
630
631	skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
632	skb = rx_ring->buf[skb_idx];
633
634	ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb);
635	if (ret) {
636	rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
637	bd_ring->wp, ret);
638	goto err_sync_device;
639	}
640
641	rx_info = RTW89_PCI_RX_SKB_CB(skb);
642	if (!rx_info->fs || !rx_info->ls) {
643	rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
644	return cnt;
645	}
646
647	rtw89_chip_query_rxdesc(rtwdev, desc_info: &desc_info, data: skb->data, data_offset: rxinfo_size);
648
649	/* first segment has RX desc */
650	offset = desc_info.offset + desc_info.rxd_len;
651	for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
652	rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset);
653	rtw89_pci_release_rpp(rtwdev, rpp);
654	}
655
656	rtw89_pci_sync_skb_for_device(rtwdev, skb);
657	rtw89_pci_rxbd_increase(rx_ring, cnt: 1);
658	cnt++;
659
660	return cnt;
661
662	err_sync_device:
663	rtw89_pci_sync_skb_for_device(rtwdev, skb);
664	return 0;
665	}
666
667	static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
668	struct rtw89_pci_rx_ring *rx_ring,
669	u32 cnt)
670	{
671	struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
672	u32 release_cnt;
673
674	while (cnt) {
675	release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, max_cnt: cnt);
676	if (!release_cnt) {
677	rtw89_err(rtwdev, "failed to release TX skbs\n");
678
679	/* skip the rest RXBD bufs */
680	rtw89_pci_rxbd_increase(rx_ring, cnt);
681	break;
682	}
683
684	cnt -= release_cnt;
685	}
686
687	rtw89_write16(rtwdev, addr: bd_ring->addr.idx, data: bd_ring->wp);
688	}
689
690	static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
691	struct rtw89_pci *rtwpci, int budget)
692	{
693	struct rtw89_pci_rx_ring *rx_ring;
694	u32 cnt;
695	int work_done;
696
697	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
698
699	spin_lock_bh(lock: &rtwpci->trx_lock);
700
701	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
702	if (cnt == 0)
703	goto out_unlock;
704
705	rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
706
707	out_unlock:
708	spin_unlock_bh(lock: &rtwpci->trx_lock);
709
710	/* always release all RPQ */
711	work_done = min_t(int, cnt, budget);
712	rtwdev->napi_budget_countdown -= work_done;
713
714	return work_done;
715	}
716
717	static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
718	struct rtw89_pci *rtwpci)
719	{
720	struct rtw89_pci_rx_ring *rx_ring;
721	struct rtw89_pci_dma_ring *bd_ring;
722	u32 reg_idx;
723	u16 hw_idx, hw_idx_next, host_idx;
724	int i;
725
726	for (i = 0; i < RTW89_RXCH_NUM; i++) {
727	rx_ring = &rtwpci->rx_rings[i];
728	bd_ring = &rx_ring->bd_ring;
729
730	reg_idx = rtw89_read32(rtwdev, addr: bd_ring->addr.idx);
731	hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
732	host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
733	hw_idx_next = (hw_idx + 1) % bd_ring->len;
734
735	if (hw_idx_next == host_idx)
736	rtw89_debug(rtwdev, mask: RTW89_DBG_UNEXP, fmt: "%d RXD unavailable\n", i);
737
738	rtw89_debug(rtwdev, mask: RTW89_DBG_TXRX,
739	fmt: "%d RXD unavailable, idx=0x%08x, len=%d\n",
740	i, reg_idx, bd_ring->len);
741	}
742	}
743
744	void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
745	struct rtw89_pci *rtwpci,
746	struct rtw89_pci_isrs *isrs)
747	{
748	isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
749	isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
750	isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
751
752	rtw89_write32(rtwdev, R_AX_HISR0, data: isrs->halt_c2h_isrs);
753	rtw89_write32(rtwdev, R_AX_PCIE_HISR00, data: isrs->isrs[0]);
754	rtw89_write32(rtwdev, R_AX_PCIE_HISR10, data: isrs->isrs[1]);
755	}
756	EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
757
758	void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
759	struct rtw89_pci *rtwpci,
760	struct rtw89_pci_isrs *isrs)
761	{
762	isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
763	isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
764	rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
765	isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
766	rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
767	isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
768	rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
769
770	if (isrs->halt_c2h_isrs)
771	rtw89_write32(rtwdev, R_AX_HISR0, data: isrs->halt_c2h_isrs);
772	if (isrs->isrs[0])
773	rtw89_write32(rtwdev, R_AX_HAXI_HISR00, data: isrs->isrs[0]);
774	if (isrs->isrs[1])
775	rtw89_write32(rtwdev, R_AX_HISR1, data: isrs->isrs[1]);
776	}
777	EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
778
779	void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
780	struct rtw89_pci *rtwpci,
781	struct rtw89_pci_isrs *isrs)
782	{
783	isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
784	isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ?
785	rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
786	isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ?
787	rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
788	isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1];
789
790	if (isrs->halt_c2h_isrs)
791	rtw89_write32(rtwdev, R_BE_HISR0, data: isrs->halt_c2h_isrs);
792	if (isrs->isrs[0])
793	rtw89_write32(rtwdev, R_BE_HAXI_HISR00, data: isrs->isrs[0]);
794	if (isrs->isrs[1])
795	rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, data: isrs->isrs[1]);
796	rtw89_write32(rtwdev, R_BE_PCIE_HISR, data: isrs->ind_isrs);
797	}
798	EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2);
799
800	void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
801	{
802	rtw89_write32(rtwdev, R_AX_HIMR0, data: rtwpci->halt_c2h_intrs);
803	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, data: rtwpci->intrs[0]);
804	rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, data: rtwpci->intrs[1]);
805	}
806	EXPORT_SYMBOL(rtw89_pci_enable_intr);
807
808	void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
809	{
810	rtw89_write32(rtwdev, R_AX_HIMR0, data: 0);
811	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, data: 0);
812	rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, data: 0);
813	}
814	EXPORT_SYMBOL(rtw89_pci_disable_intr);
815
816	void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
817	{
818	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, data: rtwpci->ind_intrs);
819	rtw89_write32(rtwdev, R_AX_HIMR0, data: rtwpci->halt_c2h_intrs);
820	rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, data: rtwpci->intrs[0]);
821	rtw89_write32(rtwdev, R_AX_HIMR1, data: rtwpci->intrs[1]);
822	}
823	EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
824
825	void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
826	{
827	rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, data: 0);
828	}
829	EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
830
831	void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
832	{
833	rtw89_write32(rtwdev, R_BE_HIMR0, data: rtwpci->halt_c2h_intrs);
834	rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, data: rtwpci->intrs[0]);
835	rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, data: rtwpci->intrs[1]);
836	rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, data: rtwpci->ind_intrs);
837	}
838	EXPORT_SYMBOL(rtw89_pci_enable_intr_v2);
839
840	void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
841	{
842	rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, data: 0);
843	rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, data: 0);
844	}
845	EXPORT_SYMBOL(rtw89_pci_disable_intr_v2);
846
847	static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
848	{
849	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
850	unsigned long flags;
851
852	spin_lock_irqsave(&rtwpci->irq_lock, flags);
853	rtw89_chip_disable_intr(rtwdev, rtwpci);
854	rtw89_chip_config_intr_mask(rtwdev, cfg: RTW89_PCI_INTR_MASK_RECOVERY_START);
855	rtw89_chip_enable_intr(rtwdev, rtwpci);
856	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
857	}
858
859	static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
860	{
861	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
862	unsigned long flags;
863
864	spin_lock_irqsave(&rtwpci->irq_lock, flags);
865	rtw89_chip_disable_intr(rtwdev, rtwpci);
866	rtw89_chip_config_intr_mask(rtwdev, cfg: RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
867	rtw89_chip_enable_intr(rtwdev, rtwpci);
868	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
869	}
870
871	static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
872	{
873	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
874	int budget = NAPI_POLL_WEIGHT;
875
876	/* To prevent RXQ get stuck due to run out of budget. */
877	rtwdev->napi_budget_countdown = budget;
878
879	rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
880	rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
881	}
882
883	static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
884	{
885	struct rtw89_dev *rtwdev = dev;
886	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
887	const struct rtw89_pci_info *info = rtwdev->pci_info;
888	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
889	struct rtw89_pci_isrs isrs;
890	unsigned long flags;
891
892	spin_lock_irqsave(&rtwpci->irq_lock, flags);
893	rtw89_chip_recognize_intrs(rtwdev, rtwpci, isrs: &isrs);
894	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
895
896	if (unlikely(isrs.isrs[0] & gen_def->isr_rdu))
897	rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
898
899	if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_halt_c2h))
900	rtw89_ser_notify(rtwdev, err: rtw89_mac_get_err_status(rtwdev));
901
902	if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_wdt_timeout))
903	rtw89_ser_notify(rtwdev, err: MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
904
905	if (unlikely(rtwpci->under_recovery))
906	goto enable_intr;
907
908	if (unlikely(rtwpci->low_power)) {
909	rtw89_pci_low_power_interrupt_handler(rtwdev);
910	goto enable_intr;
911	}
912
913	if (likely(rtwpci->running)) {
914	local_bh_disable();
915	napi_schedule(n: &rtwdev->napi);
916	local_bh_enable();
917	}
918
919	return IRQ_HANDLED;
920
921	enable_intr:
922	spin_lock_irqsave(&rtwpci->irq_lock, flags);
923	if (likely(rtwpci->running))
924	rtw89_chip_enable_intr(rtwdev, rtwpci);
925	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
926	return IRQ_HANDLED;
927	}
928
929	static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
930	{
931	struct rtw89_dev *rtwdev = dev;
932	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
933	unsigned long flags;
934	irqreturn_t irqret = IRQ_WAKE_THREAD;
935
936	spin_lock_irqsave(&rtwpci->irq_lock, flags);
937
938	/* If interrupt event is on the road, it is still trigger interrupt
939	* even we have done pci_stop() to turn off IMR.
940	*/
941	if (unlikely(!rtwpci->running)) {
942	irqret = IRQ_HANDLED;
943	goto exit;
944	}
945
946	rtw89_chip_disable_intr(rtwdev, rtwpci);
947	exit:
948	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
949
950	return irqret;
951	}
952
953	#define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \
954	[RTW89_TXCH_##ch_idx] = { \
955	.num = R_##gen##_##txch##_TXBD_NUM ##v, \
956	.idx = R_##gen##_##txch##_TXBD_IDX ##v, \
957	.bdram = 0, \
958	.desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \
959	.desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \
960	}
961
962	#define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
963	[RTW89_TXCH_##txch] = { \
964	.num = R_AX_##txch##_TXBD_NUM ##v, \
965	.idx = R_AX_##txch##_TXBD_IDX ##v, \
966	.bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
967	.desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
968	.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
969	}
970
971	#define DEF_TXCHADDRS(info, txch, v...) \
972	[RTW89_TXCH_##txch] = { \
973	.num = R_AX_##txch##_TXBD_NUM, \
974	.idx = R_AX_##txch##_TXBD_IDX, \
975	.bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
976	.desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
977	.desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
978	}
979
980	#define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \
981	[RTW89_RXCH_##ch_idx] = { \
982	.num = R_##gen##_##rxch##_RXBD_NUM ##v, \
983	.idx = R_##gen##_##rxch##_RXBD_IDX ##v, \
984	.desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \
985	.desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \
986	}
987
988	const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
989	.tx = {
990	DEF_TXCHADDRS(info, ACH0),
991	DEF_TXCHADDRS(info, ACH1),
992	DEF_TXCHADDRS(info, ACH2),
993	DEF_TXCHADDRS(info, ACH3),
994	DEF_TXCHADDRS(info, ACH4),
995	DEF_TXCHADDRS(info, ACH5),
996	DEF_TXCHADDRS(info, ACH6),
997	DEF_TXCHADDRS(info, ACH7),
998	DEF_TXCHADDRS(info, CH8),
999	DEF_TXCHADDRS(info, CH9),
1000	DEF_TXCHADDRS_TYPE1(info, CH10),
1001	DEF_TXCHADDRS_TYPE1(info, CH11),
1002	DEF_TXCHADDRS(info, CH12),
1003	},
1004	.rx = {
1005	DEF_RXCHADDRS(AX, RXQ, RXQ),
1006	DEF_RXCHADDRS(AX, RPQ, RPQ),
1007	},
1008	};
1009	EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
1010
1011	const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
1012	.tx = {
1013	DEF_TXCHADDRS(info, ACH0, _V1),
1014	DEF_TXCHADDRS(info, ACH1, _V1),
1015	DEF_TXCHADDRS(info, ACH2, _V1),
1016	DEF_TXCHADDRS(info, ACH3, _V1),
1017	DEF_TXCHADDRS(info, ACH4, _V1),
1018	DEF_TXCHADDRS(info, ACH5, _V1),
1019	DEF_TXCHADDRS(info, ACH6, _V1),
1020	DEF_TXCHADDRS(info, ACH7, _V1),
1021	DEF_TXCHADDRS(info, CH8, _V1),
1022	DEF_TXCHADDRS(info, CH9, _V1),
1023	DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
1024	DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
1025	DEF_TXCHADDRS(info, CH12, _V1),
1026	},
1027	.rx = {
1028	DEF_RXCHADDRS(AX, RXQ, RXQ, _V1),
1029	DEF_RXCHADDRS(AX, RPQ, RPQ, _V1),
1030	},
1031	};
1032	EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
1033
1034	const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = {
1035	.tx = {
1036	DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1),
1037	DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1),
1038	DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1),
1039	DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1),
1040	DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1),
1041	DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1),
1042	DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1),
1043	DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1),
1044	DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1),
1045	DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1),
1046	DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1),
1047	DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1),
1048	DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1),
1049	},
1050	.rx = {
1051	DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1),
1052	DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1),
1053	},
1054	};
1055	EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be);
1056
1057	#undef DEF_TXCHADDRS_TYPE1
1058	#undef DEF_TXCHADDRS
1059	#undef DEF_RXCHADDRS
1060
1061	static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
1062	enum rtw89_tx_channel txch,
1063	const struct rtw89_pci_ch_dma_addr **addr)
1064	{
1065	const struct rtw89_pci_info *info = rtwdev->pci_info;
1066
1067	if (txch >= RTW89_TXCH_NUM)
1068	return -EINVAL;
1069
1070	*addr = &info->dma_addr_set->tx[txch];
1071
1072	return 0;
1073	}
1074
1075	static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
1076	enum rtw89_rx_channel rxch,
1077	const struct rtw89_pci_ch_dma_addr **addr)
1078	{
1079	const struct rtw89_pci_info *info = rtwdev->pci_info;
1080
1081	if (rxch >= RTW89_RXCH_NUM)
1082	return -EINVAL;
1083
1084	*addr = &info->dma_addr_set->rx[rxch];
1085
1086	return 0;
1087	}
1088
1089	static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
1090	{
1091	struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
1092
1093	/* reserved 1 desc check ring is full or not */
1094	if (bd_ring->rp > bd_ring->wp)
1095	return bd_ring->rp - bd_ring->wp - 1;
1096
1097	return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
1098	}
1099
1100	static
1101	u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
1102	{
1103	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1104	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
1105	u32 cnt;
1106
1107	spin_lock_bh(lock: &rtwpci->trx_lock);
1108	rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
1109	cnt = rtw89_pci_get_avail_txbd_num(ring: tx_ring);
1110	spin_unlock_bh(lock: &rtwpci->trx_lock);
1111
1112	return cnt;
1113	}
1114
1115	static
1116	u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
1117	u8 txch)
1118	{
1119	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1120	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1121	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1122	u32 cnt;
1123
1124	spin_lock_bh(lock: &rtwpci->trx_lock);
1125	cnt = rtw89_pci_get_avail_txbd_num(ring: tx_ring);
1126	if (txch != RTW89_TXCH_CH12)
1127	cnt = min(cnt, wd_ring->curr_num);
1128	spin_unlock_bh(lock: &rtwpci->trx_lock);
1129
1130	return cnt;
1131	}
1132
1133	static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1134	u8 txch)
1135	{
1136	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1137	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1138	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1139	const struct rtw89_chip_info *chip = rtwdev->chip;
1140	u32 bd_cnt, wd_cnt, min_cnt = 0;
1141	struct rtw89_pci_rx_ring *rx_ring;
1142	enum rtw89_debug_mask debug_mask;
1143	u32 cnt;
1144
1145	rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
1146
1147	spin_lock_bh(lock: &rtwpci->trx_lock);
1148	bd_cnt = rtw89_pci_get_avail_txbd_num(ring: tx_ring);
1149	wd_cnt = wd_ring->curr_num;
1150
1151	if (wd_cnt == 0 || bd_cnt == 0) {
1152	cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
1153	if (cnt)
1154	rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
1155	else if (wd_cnt == 0)
1156	goto out_unlock;
1157
1158	bd_cnt = rtw89_pci_get_avail_txbd_num(ring: tx_ring);
1159	if (bd_cnt == 0)
1160	rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
1161	}
1162
1163	bd_cnt = rtw89_pci_get_avail_txbd_num(ring: tx_ring);
1164	wd_cnt = wd_ring->curr_num;
1165	min_cnt = min(bd_cnt, wd_cnt);
1166	if (min_cnt == 0) {
1167	/* This message can be frequently shown in low power mode or
1168	* high traffic with small FIFO chips, and we have recognized it as normal
1169	* behavior, so print with mask RTW89_DBG_TXRX in these situations.
1170	*/
1171	if (rtwpci->low_power || chip->small_fifo_size)
1172	debug_mask = RTW89_DBG_TXRX;
1173	else
1174	debug_mask = RTW89_DBG_UNEXP;
1175
1176	rtw89_debug(rtwdev, mask: debug_mask,
1177	fmt: "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
1178	wd_cnt, bd_cnt);
1179	}
1180
1181	out_unlock:
1182	spin_unlock_bh(lock: &rtwpci->trx_lock);
1183
1184	return min_cnt;
1185	}
1186
1187	static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1188	u8 txch)
1189	{
1190	if (rtwdev->hci.paused)
1191	return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1192
1193	if (txch == RTW89_TXCH_CH12)
1194	return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1195
1196	return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1197	}
1198
1199	static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1200	{
1201	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1202	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1203	u32 host_idx, addr;
1204
1205	spin_lock_bh(lock: &rtwpci->trx_lock);
1206
1207	addr = bd_ring->addr.idx;
1208	host_idx = bd_ring->wp;
1209	rtw89_write16(rtwdev, addr, data: host_idx);
1210
1211	spin_unlock_bh(lock: &rtwpci->trx_lock);
1212	}
1213
1214	static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1215	int n_txbd)
1216	{
1217	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1218	u32 host_idx, len;
1219
1220	len = bd_ring->len;
1221	host_idx = bd_ring->wp + n_txbd;
1222	host_idx = host_idx < len ? host_idx : host_idx - len;
1223
1224	bd_ring->wp = host_idx;
1225	}
1226
1227	static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1228	{
1229	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1230	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1231
1232	if (rtwdev->hci.paused) {
1233	set_bit(nr: txch, addr: rtwpci->kick_map);
1234	return;
1235	}
1236
1237	__rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1238	}
1239
1240	static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1241	{
1242	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1243	struct rtw89_pci_tx_ring *tx_ring;
1244	int txch;
1245
1246	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1247	if (!test_and_clear_bit(nr: txch, addr: rtwpci->kick_map))
1248	continue;
1249
1250	tx_ring = &rtwpci->tx_rings[txch];
1251	__rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1252	}
1253	}
1254
1255	static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1256	{
1257	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1258	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1259	struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1260	u32 cur_idx, cur_rp;
1261	u8 i;
1262
1263	/* Because the time taked by the I/O is a bit dynamic, it's hard to
1264	* define a reasonable fixed total timeout to use read_poll_timeout*
1265	* helper. Instead, we can ensure a reasonable polling times, so we
1266	* just use for loop with udelay here.
1267	*/
1268	for (i = 0; i < 60; i++) {
1269	cur_idx = rtw89_read32(rtwdev, addr: bd_ring->addr.idx);
1270	cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1271	if (cur_rp == bd_ring->wp)
1272	return;
1273
1274	udelay(usec: 1);
1275	}
1276
1277	if (!drop)
1278	rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1279	}
1280
1281	static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1282	bool drop)
1283	{
1284	const struct rtw89_pci_info *info = rtwdev->pci_info;
1285	u8 i;
1286
1287	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1288	/* It may be unnecessary to flush FWCMD queue. */
1289	if (i == RTW89_TXCH_CH12)
1290	continue;
1291	if (info->tx_dma_ch_mask & BIT(i))
1292	continue;
1293
1294	if (txchs & BIT(i))
1295	__pci_flush_txch(rtwdev, txch: i, drop);
1296	}
1297	}
1298
1299	static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1300	bool drop)
1301	{
1302	__rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1303	}
1304
1305	u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1306	void *txaddr_info_addr, u32 total_len,
1307	dma_addr_t dma, u8 *add_info_nr)
1308	{
1309	struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1310	__le16 option;
1311
1312	txaddr_info->length = cpu_to_le16(total_len);
1313	option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS | RTW89_PCI_ADDR_NUM(1));
1314	option |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_ADDR_HIGH_MASK);
1315	txaddr_info->option = option;
1316	txaddr_info->dma = cpu_to_le32(dma);
1317
1318	*add_info_nr = 1;
1319
1320	return sizeof(*txaddr_info);
1321	}
1322	EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1323
1324	u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1325	void *txaddr_info_addr, u32 total_len,
1326	dma_addr_t dma, u8 *add_info_nr)
1327	{
1328	struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1329	u32 remain = total_len;
1330	u32 len;
1331	u16 length_option;
1332	int n;
1333
1334	for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1335	len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1336	TXADDR_INFO_LENTHG_V1_MAX : remain;
1337	remain -= len;
1338
1339	length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1340	FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1341	FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1342	length_option |= u16_encode_bits(upper_32_bits(dma),
1343	B_PCIADDR_HIGH_SEL_V1_MASK);
1344	txaddr_info->length_opt = cpu_to_le16(length_option);
1345	txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1346	txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1347
1348	dma += len;
1349	txaddr_info++;
1350	}
1351
1352	WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1353	remain, total_len);
1354
1355	*add_info_nr = n;
1356
1357	return n * sizeof(*txaddr_info);
1358	}
1359	EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1360
1361	static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1362	struct rtw89_pci_tx_ring *tx_ring,
1363	struct rtw89_pci_tx_wd *txwd,
1364	struct rtw89_core_tx_request *tx_req)
1365	{
1366	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1367	const struct rtw89_chip_info *chip = rtwdev->chip;
1368	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1369	struct rtw89_pci_tx_wp_info *txwp_info;
1370	void *txaddr_info_addr;
1371	struct pci_dev *pdev = rtwpci->pdev;
1372	struct sk_buff *skb = tx_req->skb;
1373	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1374	struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
1375	bool en_wd_info = desc_info->en_wd_info;
1376	u32 txwd_len;
1377	u32 txwp_len;
1378	u32 txaddr_info_len;
1379	dma_addr_t dma;
1380	int ret;
1381
1382	dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1383	if (dma_mapping_error(dev: &pdev->dev, dma_addr: dma)) {
1384	rtw89_err(rtwdev, "failed to map skb dma data\n");
1385	ret = -EBUSY;
1386	goto err;
1387	}
1388
1389	tx_data->dma = dma;
1390	rcu_assign_pointer(skb_data->wait, NULL);
1391
1392	txwp_len = sizeof(*txwp_info);
1393	txwd_len = chip->txwd_body_size;
1394	txwd_len += en_wd_info ? chip->txwd_info_size : 0;
1395
1396	txwp_info = txwd->vaddr + txwd_len;
1397	txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1398	txwp_info->seq1 = 0;
1399	txwp_info->seq2 = 0;
1400	txwp_info->seq3 = 0;
1401
1402	tx_ring->tx_cnt++;
1403	txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1404	txaddr_info_len =
1405	rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, total_len: skb->len,
1406	dma, add_info_nr: &desc_info->addr_info_nr);
1407
1408	txwd->len = txwd_len + txwp_len + txaddr_info_len;
1409
1410	rtw89_chip_fill_txdesc(rtwdev, desc_info, txdesc: txwd->vaddr);
1411
1412	skb_queue_tail(list: &txwd->queue, newsk: skb);
1413
1414	return 0;
1415
1416	err:
1417	return ret;
1418	}
1419
1420	static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1421	struct rtw89_pci_tx_ring *tx_ring,
1422	struct rtw89_pci_tx_bd_32 *txbd,
1423	struct rtw89_core_tx_request *tx_req)
1424	{
1425	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1426	const struct rtw89_chip_info *chip = rtwdev->chip;
1427	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1428	void *txdesc;
1429	int txdesc_size = chip->h2c_desc_size;
1430	struct pci_dev *pdev = rtwpci->pdev;
1431	struct sk_buff *skb = tx_req->skb;
1432	struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1433	dma_addr_t dma;
1434	__le16 opt;
1435
1436	txdesc = skb_push(skb, len: txdesc_size);
1437	memset(txdesc, 0, txdesc_size);
1438	rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1439
1440	dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1441	if (dma_mapping_error(dev: &pdev->dev, dma_addr: dma)) {
1442	rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1443	return -EBUSY;
1444	}
1445
1446	tx_data->dma = dma;
1447	opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS);
1448	opt |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_TXBD_OPT_DMA_HI);
1449	txbd->opt = opt;
1450	txbd->length = cpu_to_le16(skb->len);
1451	txbd->dma = cpu_to_le32(tx_data->dma);
1452	skb_queue_tail(list: &rtwpci->h2c_queue, newsk: skb);
1453
1454	rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, n_txbd: 1);
1455
1456	return 0;
1457	}
1458
1459	static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1460	struct rtw89_pci_tx_ring *tx_ring,
1461	struct rtw89_pci_tx_bd_32 *txbd,
1462	struct rtw89_core_tx_request *tx_req)
1463	{
1464	struct rtw89_pci_tx_wd *txwd;
1465	__le16 opt;
1466	int ret;
1467
1468	/* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1469	* buffer with WD BODY only. So here we don't need to check the free
1470	* pages of the wd ring.
1471	*/
1472	if (tx_ring->txch == RTW89_TXCH_CH12)
1473	return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1474
1475	txwd = rtw89_pci_dequeue_txwd(tx_ring);
1476	if (!txwd) {
1477	rtw89_err(rtwdev, "no available TXWD\n");
1478	ret = -ENOSPC;
1479	goto err;
1480	}
1481
1482	ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1483	if (ret) {
1484	rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1485	goto err_enqueue_wd;
1486	}
1487
1488	list_add_tail(new: &txwd->list, head: &tx_ring->busy_pages);
1489
1490	opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS);
1491	opt |= le16_encode_bits(upper_32_bits(txwd->paddr), RTW89_PCI_TXBD_OPT_DMA_HI);
1492	txbd->opt = opt;
1493	txbd->length = cpu_to_le16(txwd->len);
1494	txbd->dma = cpu_to_le32(txwd->paddr);
1495
1496	rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, n_txbd: 1);
1497
1498	return 0;
1499
1500	err_enqueue_wd:
1501	rtw89_pci_enqueue_txwd(tx_ring, txwd);
1502	err:
1503	return ret;
1504	}
1505
1506	static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1507	u8 txch)
1508	{
1509	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1510	struct rtw89_pci_tx_ring *tx_ring;
1511	struct rtw89_pci_tx_bd_32 *txbd;
1512	u32 n_avail_txbd;
1513	int ret = 0;
1514
1515	/* check the tx type and dma channel for fw cmd queue */
1516	if ((txch == RTW89_TXCH_CH12 ||
1517	tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1518	(txch != RTW89_TXCH_CH12 ||
1519	tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1520	rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1521	return -EINVAL;
1522	}
1523
1524	tx_ring = &rtwpci->tx_rings[txch];
1525	spin_lock_bh(lock: &rtwpci->trx_lock);
1526
1527	n_avail_txbd = rtw89_pci_get_avail_txbd_num(ring: tx_ring);
1528	if (n_avail_txbd == 0) {
1529	rtw89_err(rtwdev, "no available TXBD\n");
1530	ret = -ENOSPC;
1531	goto err_unlock;
1532	}
1533
1534	txbd = rtw89_pci_get_next_txbd(tx_ring);
1535	ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1536	if (ret) {
1537	rtw89_err(rtwdev, "failed to submit TXBD\n");
1538	goto err_unlock;
1539	}
1540
1541	spin_unlock_bh(lock: &rtwpci->trx_lock);
1542	return 0;
1543
1544	err_unlock:
1545	spin_unlock_bh(lock: &rtwpci->trx_lock);
1546	return ret;
1547	}
1548
1549	static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1550	{
1551	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1552	int ret;
1553
1554	ret = rtw89_pci_tx_write(rtwdev, tx_req, txch: desc_info->ch_dma);
1555	if (ret) {
1556	rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1557	return ret;
1558	}
1559
1560	return 0;
1561	}
1562
1563	const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = {
1564	[RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1565	[RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1566	[RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1567	[RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1568	[RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1569	[RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1570	[RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1571	[RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1572	[RTW89_TXCH_CH8] = {.start_idx = 40, .max_num = 5, .min_num = 1},
1573	[RTW89_TXCH_CH9] = {.start_idx = 45, .max_num = 5, .min_num = 1},
1574	[RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1575	[RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1576	[RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1577	};
1578	EXPORT_SYMBOL(rtw89_bd_ram_table_dual);
1579
1580	const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = {
1581	[RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1582	[RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1583	[RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1584	[RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1585	[RTW89_TXCH_CH8] = {.start_idx = 20, .max_num = 4, .min_num = 1},
1586	[RTW89_TXCH_CH9] = {.start_idx = 24, .max_num = 4, .min_num = 1},
1587	[RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1},
1588	};
1589	EXPORT_SYMBOL(rtw89_bd_ram_table_single);
1590
1591	static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev)
1592	{
1593	const struct rtw89_pci_info *info = rtwdev->pci_info;
1594	u32 addr = info->wp_sel_addr;
1595	u32 val;
1596	int i;
1597
1598	if (!info->wp_sel_addr)
1599	return;
1600
1601	for (i = 0; i < 16; i += 4) {
1602	val = u32_encode_bits(v: i + 0, MASKBYTE0) |
1603	u32_encode_bits(v: i + 1, MASKBYTE1) |
1604	u32_encode_bits(v: i + 2, MASKBYTE2) |
1605	u32_encode_bits(v: i + 3, MASKBYTE3);
1606	rtw89_write32(rtwdev, addr: addr + i, data: val);
1607	}
1608	}
1609
1610	static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1611	{
1612	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1613	const struct rtw89_pci_info *info = rtwdev->pci_info;
1614	const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table;
1615	struct rtw89_pci_tx_ring *tx_ring;
1616	struct rtw89_pci_rx_ring *rx_ring;
1617	struct rtw89_pci_dma_ring *bd_ring;
1618	const struct rtw89_pci_bd_ram *bd_ram;
1619	u32 addr_num;
1620	u32 addr_idx;
1621	u32 addr_bdram;
1622	u32 addr_desa_l;
1623	u32 val32;
1624	int i;
1625
1626	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1627	if (info->tx_dma_ch_mask & BIT(i))
1628	continue;
1629
1630	tx_ring = &rtwpci->tx_rings[i];
1631	bd_ring = &tx_ring->bd_ring;
1632	bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL;
1633	addr_num = bd_ring->addr.num;
1634	addr_bdram = bd_ring->addr.bdram;
1635	addr_desa_l = bd_ring->addr.desa_l;
1636	bd_ring->wp = 0;
1637	bd_ring->rp = 0;
1638
1639	rtw89_write16(rtwdev, addr: addr_num, data: bd_ring->len);
1640	if (addr_bdram && bd_ram) {
1641	val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1642	FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1643	FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1644
1645	rtw89_write32(rtwdev, addr: addr_bdram, data: val32);
1646	}
1647	rtw89_write32(rtwdev, addr: addr_desa_l, data: bd_ring->dma);
1648	rtw89_write32(rtwdev, addr: addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1649	}
1650
1651	for (i = 0; i < RTW89_RXCH_NUM; i++) {
1652	rx_ring = &rtwpci->rx_rings[i];
1653	bd_ring = &rx_ring->bd_ring;
1654	addr_num = bd_ring->addr.num;
1655	addr_idx = bd_ring->addr.idx;
1656	addr_desa_l = bd_ring->addr.desa_l;
1657	if (info->rx_ring_eq_is_full)
1658	bd_ring->wp = bd_ring->len - 1;
1659	else
1660	bd_ring->wp = 0;
1661	bd_ring->rp = 0;
1662	rx_ring->diliver_skb = NULL;
1663	rx_ring->diliver_desc.ready = false;
1664	rx_ring->target_rx_tag = 0;
1665
1666	rtw89_write16(rtwdev, addr: addr_num, data: bd_ring->len);
1667	rtw89_write32(rtwdev, addr: addr_desa_l, data: bd_ring->dma);
1668	rtw89_write32(rtwdev, addr: addr_desa_l + 4, upper_32_bits(bd_ring->dma));
1669
1670	if (info->rx_ring_eq_is_full)
1671	rtw89_write16(rtwdev, addr: addr_idx, data: bd_ring->wp);
1672	}
1673
1674	rtw89_pci_init_wp_16sel(rtwdev);
1675	}
1676
1677	static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1678	struct rtw89_pci_tx_ring *tx_ring)
1679	{
1680	rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1681	rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1682	}
1683
1684	void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1685	{
1686	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1687	const struct rtw89_pci_info *info = rtwdev->pci_info;
1688	int txch;
1689
1690	rtw89_pci_reset_trx_rings(rtwdev);
1691
1692	spin_lock_bh(lock: &rtwpci->trx_lock);
1693	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1694	if (info->tx_dma_ch_mask & BIT(txch))
1695	continue;
1696	if (txch == RTW89_TXCH_CH12) {
1697	rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1698	cnt: skb_queue_len(list_: &rtwpci->h2c_queue), release_all: true);
1699	continue;
1700	}
1701	rtw89_pci_release_tx_ring(rtwdev, tx_ring: &rtwpci->tx_rings[txch]);
1702	}
1703	spin_unlock_bh(lock: &rtwpci->trx_lock);
1704	}
1705
1706	static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1707	{
1708	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1709	unsigned long flags;
1710
1711	spin_lock_irqsave(&rtwpci->irq_lock, flags);
1712	rtwpci->running = true;
1713	rtw89_chip_enable_intr(rtwdev, rtwpci);
1714	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
1715	}
1716
1717	static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1718	{
1719	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1720	unsigned long flags;
1721
1722	spin_lock_irqsave(&rtwpci->irq_lock, flags);
1723	rtwpci->running = false;
1724	rtw89_chip_disable_intr(rtwdev, rtwpci);
1725	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
1726	}
1727
1728	static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1729	{
1730	rtw89_core_napi_start(rtwdev);
1731	rtw89_pci_enable_intr_lock(rtwdev);
1732
1733	return 0;
1734	}
1735
1736	static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1737	{
1738	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1739	struct pci_dev *pdev = rtwpci->pdev;
1740
1741	rtw89_pci_disable_intr_lock(rtwdev);
1742	synchronize_irq(irq: pdev->irq);
1743	rtw89_core_napi_stop(rtwdev);
1744	}
1745
1746	static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1747	{
1748	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1749	struct pci_dev *pdev = rtwpci->pdev;
1750
1751	if (pause) {
1752	rtw89_pci_disable_intr_lock(rtwdev);
1753	synchronize_irq(irq: pdev->irq);
1754	if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1755	napi_synchronize(n: &rtwdev->napi);
1756	} else {
1757	rtw89_pci_enable_intr_lock(rtwdev);
1758	rtw89_pci_tx_kick_off_pending(rtwdev);
1759	}
1760	}
1761
1762	static
1763	void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1764	{
1765	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1766	const struct rtw89_pci_info *info = rtwdev->pci_info;
1767	const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1768	const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1769	struct rtw89_pci_tx_ring *tx_ring;
1770	struct rtw89_pci_rx_ring *rx_ring;
1771	int i;
1772
1773	if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1774	return;
1775
1776	for (i = 0; i < RTW89_TXCH_NUM; i++) {
1777	tx_ring = &rtwpci->tx_rings[i];
1778	tx_ring->bd_ring.addr.idx = low_power ?
1779	bd_idx_addr->tx_bd_addrs[i] :
1780	dma_addr_set->tx[i].idx;
1781	}
1782
1783	for (i = 0; i < RTW89_RXCH_NUM; i++) {
1784	rx_ring = &rtwpci->rx_rings[i];
1785	rx_ring->bd_ring.addr.idx = low_power ?
1786	bd_idx_addr->rx_bd_addrs[i] :
1787	dma_addr_set->rx[i].idx;
1788	}
1789	}
1790
1791	static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1792	{
1793	enum rtw89_pci_intr_mask_cfg cfg;
1794
1795	WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1796
1797	cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
1798	rtw89_chip_config_intr_mask(rtwdev, cfg);
1799	rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1800	}
1801
1802	static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1803
1804	static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1805	{
1806	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1807	u32 val = readl(addr: rtwpci->mmap + addr);
1808	int count;
1809
1810	for (count = 0; ; count++) {
1811	if (val != RTW89_R32_DEAD)
1812	return val;
1813	if (count >= MAC_REG_POOL_COUNT) {
1814	rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1815	return RTW89_R32_DEAD;
1816	}
1817	rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1818	val = readl(addr: rtwpci->mmap + addr);
1819	}
1820
1821	return val;
1822	}
1823
1824	static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1825	{
1826	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1827	u32 addr32, val32, shift;
1828
1829	if (!ACCESS_CMAC(addr))
1830	return readb(addr: rtwpci->mmap + addr);
1831
1832	addr32 = addr & ~0x3;
1833	shift = (addr & 0x3) * 8;
1834	val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr: addr32);
1835	return val32 >> shift;
1836	}
1837
1838	static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1839	{
1840	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1841	u32 addr32, val32, shift;
1842
1843	if (!ACCESS_CMAC(addr))
1844	return readw(addr: rtwpci->mmap + addr);
1845
1846	addr32 = addr & ~0x3;
1847	shift = (addr & 0x3) * 8;
1848	val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr: addr32);
1849	return val32 >> shift;
1850	}
1851
1852	static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1853	{
1854	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1855
1856	if (!ACCESS_CMAC(addr))
1857	return readl(addr: rtwpci->mmap + addr);
1858
1859	return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1860	}
1861
1862	static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1863	{
1864	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1865
1866	writeb(val: data, addr: rtwpci->mmap + addr);
1867	}
1868
1869	static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1870	{
1871	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1872
1873	writew(val: data, addr: rtwpci->mmap + addr);
1874	}
1875
1876	static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1877	{
1878	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1879
1880	writel(val: data, addr: rtwpci->mmap + addr);
1881	}
1882
1883	static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1884	{
1885	const struct rtw89_pci_info *info = rtwdev->pci_info;
1886
1887	if (enable)
1888	rtw89_write32_set(rtwdev, addr: info->init_cfg_reg,
1889	bit: info->rxhci_en_bit | info->txhci_en_bit);
1890	else
1891	rtw89_write32_clr(rtwdev, addr: info->init_cfg_reg,
1892	bit: info->rxhci_en_bit | info->txhci_en_bit);
1893	}
1894
1895	static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1896	{
1897	const struct rtw89_pci_info *info = rtwdev->pci_info;
1898	const struct rtw89_reg_def *reg = &info->dma_io_stop;
1899
1900	if (enable)
1901	rtw89_write32_clr(rtwdev, addr: reg->addr, bit: reg->mask);
1902	else
1903	rtw89_write32_set(rtwdev, addr: reg->addr, bit: reg->mask);
1904	}
1905
1906	void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1907	{
1908	rtw89_pci_ctrl_dma_io(rtwdev, enable);
1909	rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1910	}
1911
1912	static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1913	{
1914	u16 val;
1915
1916	rtw89_write8(rtwdev, R_AX_MDIO_CFG, data: addr & 0x1F);
1917
1918	val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1919	switch (speed) {
1920	case PCIE_PHY_GEN1:
1921	if (addr < 0x20)
1922	val = u16_replace_bits(old: val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
1923	else
1924	val = u16_replace_bits(old: val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
1925	break;
1926	case PCIE_PHY_GEN2:
1927	if (addr < 0x20)
1928	val = u16_replace_bits(old: val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
1929	else
1930	val = u16_replace_bits(old: val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
1931	break;
1932	default:
1933	rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
1934	return -EINVAL;
1935	}
1936	rtw89_write16(rtwdev, R_AX_MDIO_CFG, data: val);
1937	rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, bit: rw_bit);
1938
1939	return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
1940	false, rtwdev, R_AX_MDIO_CFG);
1941	}
1942
1943	static int
1944	rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
1945	{
1946	int ret;
1947
1948	ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
1949	if (ret) {
1950	rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
1951	return ret;
1952	}
1953	*val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
1954
1955	return 0;
1956	}
1957
1958	static int
1959	rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
1960	{
1961	int ret;
1962
1963	rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
1964	ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
1965	if (ret) {
1966	rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
1967	return ret;
1968	}
1969
1970	return 0;
1971	}
1972
1973	static int
1974	rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
1975	{
1976	u32 shift;
1977	int ret;
1978	u16 val;
1979
1980	ret = rtw89_read16_mdio(rtwdev, addr, speed, val: &val);
1981	if (ret)
1982	return ret;
1983
1984	shift = __ffs(mask);
1985	val &= ~mask;
1986	val |= ((data << shift) & mask);
1987
1988	ret = rtw89_write16_mdio(rtwdev, addr, data: val, speed);
1989	if (ret)
1990	return ret;
1991
1992	return 0;
1993	}
1994
1995	static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1996	{
1997	int ret;
1998	u16 val;
1999
2000	ret = rtw89_read16_mdio(rtwdev, addr, speed, val: &val);
2001	if (ret)
2002	return ret;
2003	ret = rtw89_write16_mdio(rtwdev, addr, data: val | mask, speed);
2004	if (ret)
2005	return ret;
2006
2007	return 0;
2008	}
2009
2010	static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
2011	{
2012	int ret;
2013	u16 val;
2014
2015	ret = rtw89_read16_mdio(rtwdev, addr, speed, val: &val);
2016	if (ret)
2017	return ret;
2018	ret = rtw89_write16_mdio(rtwdev, addr, data: val & ~mask, speed);
2019	if (ret)
2020	return ret;
2021
2022	return 0;
2023	}
2024
2025	static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data)
2026	{
2027	u16 addr_2lsb = addr & B_AX_DBI_2LSB;
2028	u16 write_addr;
2029	u8 flag;
2030	int ret;
2031
2032	write_addr = addr & B_AX_DBI_ADDR_MSK;
2033	write_addr |= u16_encode_bits(BIT(addr_2lsb), B_AX_DBI_WREN_MSK);
2034	rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data);
2035	rtw89_write16(rtwdev, R_AX_DBI_FLAG, data: write_addr);
2036	rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16);
2037
2038	ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10,
2039	10 * RTW89_PCI_WR_RETRY_CNT, false,
2040	rtwdev, R_AX_DBI_FLAG + 2);
2041	if (ret)
2042	rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n",
2043	addr);
2044
2045	return ret;
2046	}
2047
2048	static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value)
2049	{
2050	u16 read_addr = addr & B_AX_DBI_ADDR_MSK;
2051	u8 flag;
2052	int ret;
2053
2054	rtw89_write16(rtwdev, R_AX_DBI_FLAG, data: read_addr);
2055	rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16);
2056
2057	ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10,
2058	10 * RTW89_PCI_WR_RETRY_CNT, false,
2059	rtwdev, R_AX_DBI_FLAG + 2);
2060	if (ret) {
2061	rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n",
2062	addr);
2063	return ret;
2064	}
2065
2066	read_addr = R_AX_DBI_RDATA + (addr & 3);
2067	*value = rtw89_read8(rtwdev, addr: read_addr);
2068
2069	return 0;
2070	}
2071
2072	static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2073	u8 data)
2074	{
2075	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2076	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2077	struct pci_dev *pdev = rtwpci->pdev;
2078	int ret;
2079
2080	ret = pci_write_config_byte(dev: pdev, where: addr, val: data);
2081	if (!ret)
2082	return 0;
2083
2084	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2085	ret = rtw89_dbi_write8(rtwdev, addr, data);
2086
2087	return ret;
2088	}
2089
2090	static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
2091	u8 *value)
2092	{
2093	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2094	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2095	struct pci_dev *pdev = rtwpci->pdev;
2096	int ret;
2097
2098	ret = pci_read_config_byte(dev: pdev, where: addr, val: value);
2099	if (!ret)
2100	return 0;
2101
2102	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev))
2103	ret = rtw89_dbi_read8(rtwdev, addr, value);
2104
2105	return ret;
2106	}
2107
2108	static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
2109	u8 bit)
2110	{
2111	u8 value;
2112	int ret;
2113
2114	ret = rtw89_pci_read_config_byte(rtwdev, addr, value: &value);
2115	if (ret)
2116	return ret;
2117
2118	value |= bit;
2119	ret = rtw89_pci_write_config_byte(rtwdev, addr, data: value);
2120
2121	return ret;
2122	}
2123
2124	static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
2125	u8 bit)
2126	{
2127	u8 value;
2128	int ret;
2129
2130	ret = rtw89_pci_read_config_byte(rtwdev, addr, value: &value);
2131	if (ret)
2132	return ret;
2133
2134	value &= ~bit;
2135	ret = rtw89_pci_write_config_byte(rtwdev, addr, data: value);
2136
2137	return ret;
2138	}
2139
2140	static int
2141	__get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
2142	{
2143	u16 val, tar;
2144	int ret;
2145
2146	/* Enable counter */
2147	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, speed: phy_rate, val: &val);
2148	if (ret)
2149	return ret;
2150	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, data: val & ~B_AX_CLK_CALIB_EN,
2151	speed: phy_rate);
2152	if (ret)
2153	return ret;
2154	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, data: val | B_AX_CLK_CALIB_EN,
2155	speed: phy_rate);
2156	if (ret)
2157	return ret;
2158
2159	fsleep(usecs: 300);
2160
2161	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, speed: phy_rate, val: &tar);
2162	if (ret)
2163	return ret;
2164	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, data: val & ~B_AX_CLK_CALIB_EN,
2165	speed: phy_rate);
2166	if (ret)
2167	return ret;
2168
2169	tar = tar & 0x0FFF;
2170	if (tar == 0 || tar == 0x0FFF) {
2171	rtw89_err(rtwdev, "[ERR]Get target failed.\n");
2172	return -EINVAL;
2173	}
2174
2175	*target = tar;
2176
2177	return 0;
2178	}
2179
2180	static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
2181	{
2182	int ret;
2183
2184	if (!rtw89_is_rtl885xb(rtwdev))
2185	return 0;
2186
2187	ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
2188	PCIE_AUTOK_4, speed: PCIE_PHY_GEN1);
2189	return ret;
2190	}
2191
2192	static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
2193	{
2194	enum rtw89_pcie_phy phy_rate;
2195	u16 val16, mgn_set, div_set, tar;
2196	u8 val8, bdr_ori;
2197	bool l1_flag = false;
2198	int ret = 0;
2199
2200	if (!rtw89_is_rtl885xb(rtwdev))
2201	return 0;
2202
2203	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, value: &val8);
2204	if (ret) {
2205	rtw89_err(rtwdev, "[ERR]pci config read %X\n",
2206	RTW89_PCIE_PHY_RATE);
2207	return ret;
2208	}
2209
2210	if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
2211	phy_rate = PCIE_PHY_GEN1;
2212	} else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
2213	phy_rate = PCIE_PHY_GEN2;
2214	} else {
2215	rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
2216	return -EOPNOTSUPP;
2217	}
2218	/* Disable L1BD */
2219	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, value: &bdr_ori);
2220	if (ret) {
2221	rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
2222	return ret;
2223	}
2224
2225	if (bdr_ori & RTW89_PCIE_BIT_L1) {
2226	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2227	data: bdr_ori & ~RTW89_PCIE_BIT_L1);
2228	if (ret) {
2229	rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2230	RTW89_PCIE_L1_CTRL);
2231	return ret;
2232	}
2233	l1_flag = true;
2234	}
2235
2236	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, speed: phy_rate, val: &val16);
2237	if (ret) {
2238	rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2239	goto end;
2240	}
2241
2242	if (val16 & B_AX_CALIB_EN) {
2243	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
2244	data: val16 & ~B_AX_CALIB_EN, speed: phy_rate);
2245	if (ret) {
2246	rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2247	goto end;
2248	}
2249	}
2250
2251	if (!autook_en)
2252	goto end;
2253	/* Set div */
2254	ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, speed: phy_rate);
2255	if (ret) {
2256	rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2257	goto end;
2258	}
2259
2260	/* Obtain div and margin */
2261	ret = __get_target(rtwdev, target: &tar, phy_rate);
2262	if (ret) {
2263	rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2264	goto end;
2265	}
2266
2267	mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
2268
2269	if (mgn_set >= 128) {
2270	div_set = 0x0003;
2271	mgn_set = 0x000F;
2272	} else if (mgn_set >= 64) {
2273	div_set = 0x0003;
2274	mgn_set >>= 3;
2275	} else if (mgn_set >= 32) {
2276	div_set = 0x0002;
2277	mgn_set >>= 2;
2278	} else if (mgn_set >= 16) {
2279	div_set = 0x0001;
2280	mgn_set >>= 1;
2281	} else if (mgn_set == 0) {
2282	rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2283	goto end;
2284	} else {
2285	div_set = 0x0000;
2286	}
2287
2288	ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, speed: phy_rate, val: &val16);
2289	if (ret) {
2290	rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2291	goto end;
2292	}
2293
2294	val16 |= u16_encode_bits(v: div_set, B_AX_DIV);
2295
2296	ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, data: val16, speed: phy_rate);
2297	if (ret) {
2298	rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2299	goto end;
2300	}
2301
2302	ret = __get_target(rtwdev, target: &tar, phy_rate);
2303	if (ret) {
2304	rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2305	goto end;
2306	}
2307
2308	rtw89_debug(rtwdev, mask: RTW89_DBG_HCI, fmt: "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2309	tar, div_set, mgn_set);
2310	ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2311	data: (tar & 0x0FFF) | (mgn_set << 12), speed: phy_rate);
2312	if (ret) {
2313	rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2314	goto end;
2315	}
2316
2317	/* Enable function */
2318	ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, speed: phy_rate);
2319	if (ret) {
2320	rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2321	goto end;
2322	}
2323
2324	/* CLK delay = 0 */
2325	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2326	data: PCIE_CLKDLY_HW_0);
2327
2328	end:
2329	/* Set L1BD to ori */
2330	if (l1_flag) {
2331	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2332	data: bdr_ori);
2333	if (ret) {
2334	rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2335	RTW89_PCIE_L1_CTRL);
2336	return ret;
2337	}
2338	}
2339
2340	return ret;
2341	}
2342
2343	static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2344	{
2345	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2346	int ret;
2347
2348	if (chip_id == RTL8852A) {
2349	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2350	speed: PCIE_PHY_GEN1);
2351	if (ret)
2352	return ret;
2353	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2354	speed: PCIE_PHY_GEN2);
2355	if (ret)
2356	return ret;
2357	} else if (chip_id == RTL8852C) {
2358	rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2359	B_AX_DEGLITCH);
2360	rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2361	B_AX_DEGLITCH);
2362	}
2363
2364	return 0;
2365	}
2366
2367	static void rtw89_pci_disable_eq_ax(struct rtw89_dev *rtwdev)
2368	{
2369	u16 g1_oobs, g2_oobs;
2370	u32 backup_aspm;
2371	u32 phy_offset;
2372	u16 offset_cal;
2373	u16 oobs_val;
2374	int ret;
2375	u8 gen;
2376
2377	if (rtwdev->chip->chip_id != RTL8852C)
2378	return;
2379
2380	g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2381	RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL);
2382	g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 +
2383	RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL);
2384	if (g1_oobs && g2_oobs)
2385	return;
2386
2387	backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1);
2388	rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
2389
2390	ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, phy_offset: &phy_offset);
2391	if (ret)
2392	goto out;
2393
2394	rtw89_write16_set(rtwdev, addr: phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN);
2395	rtw89_write16(rtwdev, addr: phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL);
2396	rtw89_write16_set(rtwdev, addr: phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL);
2397
2398	oobs_val = rtw89_read16_mask(rtwdev, addr: phy_offset + RAC_ANA1F * RAC_MULT,
2399	OOBS_LEVEL_MASK);
2400
2401	rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT,
2402	OOBS_SEN_MASK, data: oobs_val);
2403	rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT,
2404	BAC_OOBS_SEL);
2405
2406	rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT,
2407	OOBS_SEN_MASK, data: oobs_val);
2408	rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT,
2409	BAC_OOBS_SEL);
2410
2411	/* offset K */
2412	for (gen = 1; gen <= 2; gen++) {
2413	phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 :
2414	R_RAC_DIRECT_OFFSET_G2;
2415
2416	rtw89_write16_clr(rtwdev, addr: phy_offset + RAC_ANA19 * RAC_MULT,
2417	B_PCIE_BIT_RD_SEL);
2418	}
2419
2420	offset_cal = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 +
2421	RAC_ANA1F * RAC_MULT, OFFSET_CAL_MASK);
2422
2423	for (gen = 1; gen <= 2; gen++) {
2424	phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 :
2425	R_RAC_DIRECT_OFFSET_G2;
2426
2427	rtw89_write16_mask(rtwdev, addr: phy_offset + RAC_ANA0B * RAC_MULT,
2428	MANUAL_LVL_MASK, data: offset_cal);
2429	rtw89_write16_clr(rtwdev, addr: phy_offset + RAC_ANA0D * RAC_MULT,
2430	OFFSET_CAL_MODE);
2431	}
2432
2433	out:
2434	rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, data: backup_aspm);
2435	}
2436
2437	static void rtw89_pci_ber(struct rtw89_dev *rtwdev)
2438	{
2439	u32 phy_offset;
2440
2441	if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks))
2442	return;
2443
2444	phy_offset = R_RAC_DIRECT_OFFSET_G1;
2445	rtw89_write16(rtwdev, addr: phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL);
2446	rtw89_write16(rtwdev, addr: phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2447
2448	phy_offset = R_RAC_DIRECT_OFFSET_G2;
2449	rtw89_write16(rtwdev, addr: phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL);
2450	rtw89_write16(rtwdev, addr: phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL);
2451	}
2452
2453	static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2454	{
2455	if (rtwdev->chip->chip_id != RTL8852A)
2456	return;
2457
2458	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2459	}
2460
2461	static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2462	{
2463	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2464
2465	if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2466	return;
2467
2468	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2469	}
2470
2471	static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2472	{
2473	int ret;
2474
2475	if (rtwdev->chip->chip_id != RTL8852A)
2476	return 0;
2477
2478	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2479	speed: PCIE_PHY_GEN1);
2480	if (ret)
2481	return ret;
2482
2483	ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2484	speed: PCIE_PHY_GEN2);
2485	if (ret)
2486	return ret;
2487
2488	return 0;
2489	}
2490
2491	static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2492	{
2493	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2494
2495	if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev))
2496	return;
2497
2498	rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2499	}
2500
2501	static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2502	{
2503	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2504
2505	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2506	rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2507	B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2508	rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2509	B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2510	} else if (rtwdev->chip->chip_id == RTL8852C) {
2511	rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2512	B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2513	}
2514	}
2515
2516	static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2517	{
2518	if (!rtw89_is_rtl885xb(rtwdev))
2519	return 0;
2520
2521	return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2522	PCIE_DPHY_DLY_25US, speed: PCIE_PHY_GEN1);
2523	}
2524
2525	static void rtw89_pci_power_wake_ax(struct rtw89_dev *rtwdev, bool pwr_up)
2526	{
2527	if (pwr_up)
2528	rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2529	else
2530	rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2531	}
2532
2533	static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2534	{
2535	if (rtwdev->chip->chip_id != RTL8852C)
2536	return;
2537
2538	rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2539	rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2540	}
2541
2542	static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2543	{
2544	if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2545	return;
2546
2547	rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2548	}
2549
2550	static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2551	{
2552	if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2553	return;
2554
2555	rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2556	B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2557	rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2558	rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2559	B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2560	}
2561
2562	static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2563	{
2564	if (rtwdev->chip->chip_id != RTL8852C)
2565	return;
2566
2567	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2568	}
2569
2570	static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2571	{
2572	if (rtwdev->chip->chip_id != RTL8852C)
2573	return;
2574
2575	rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2576	}
2577
2578	static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2579	{
2580	if (rtwdev->chip->chip_id == RTL8852C)
2581	return;
2582
2583	rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2584	B_AX_SIC_EN_FORCE_CLKREQ);
2585	}
2586
2587	static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2588	{
2589	const struct rtw89_pci_info *info = rtwdev->pci_info;
2590	u32 lbc;
2591
2592	if (rtwdev->chip->chip_id == RTL8852C)
2593	return;
2594
2595	lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2596	if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2597	lbc = u32_replace_bits(old: lbc, val: info->lbc_tmr, B_AX_LBC_TIMER);
2598	lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2599	rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, data: lbc);
2600	} else {
2601	lbc &= ~B_AX_LBC_EN;
2602	}
2603	rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, bit: lbc);
2604	}
2605
2606	static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2607	{
2608	const struct rtw89_pci_info *info = rtwdev->pci_info;
2609	u32 val32;
2610
2611	if (rtwdev->chip->chip_id != RTL8852C)
2612	return;
2613
2614	if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2615	val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2616	info->io_rcy_tmr);
2617	rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, data: val32);
2618	rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, data: val32);
2619	rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, data: val32);
2620
2621	rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2622	rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2623	rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2624	} else {
2625	rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2626	rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2627	rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2628	}
2629
2630	rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2631	}
2632
2633	static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2634	{
2635	if (rtwdev->chip->chip_id == RTL8852C)
2636	return;
2637
2638	rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2639	B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2640
2641	if (rtwdev->chip->chip_id == RTL8852A)
2642	rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2643	B_AX_EN_CHKDSC_NO_RX_STUCK);
2644	}
2645
2646	static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2647	{
2648	if (rtwdev->chip->chip_id == RTL8852C)
2649	return;
2650
2651	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2652	B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2653	}
2654
2655	static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
2656	{
2657	const struct rtw89_pci_info *info = rtwdev->pci_info;
2658	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2659	u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2660	B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2661	B_AX_CLR_CH12_IDX;
2662	u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2663	u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2664
2665	if (chip_id == RTL8852A || chip_id == RTL8852C)
2666	val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2667	B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2668	/* clear DMA indexes */
2669	rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, bit: val);
2670	if (chip_id == RTL8852A || chip_id == RTL8852C)
2671	rtw89_write32_set(rtwdev, addr: txbd_rwptr_clr2,
2672	B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2673	rtw89_write32_set(rtwdev, addr: rxbd_rwptr_clr,
2674	B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2675	}
2676
2677	static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2678	{
2679	const struct rtw89_pci_info *info = rtwdev->pci_info;
2680	u32 dma_busy1 = info->dma_busy1.addr;
2681	u32 dma_busy2 = info->dma_busy2_reg;
2682	u32 check, dma_busy;
2683	int ret;
2684
2685	check = info->dma_busy1.mask;
2686
2687	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2688	10, 100, false, rtwdev, dma_busy1);
2689	if (ret)
2690	return ret;
2691
2692	if (!dma_busy2)
2693	return 0;
2694
2695	check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2696
2697	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2698	10, 100, false, rtwdev, dma_busy2);
2699	if (ret)
2700	return ret;
2701
2702	return 0;
2703	}
2704
2705	static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev)
2706	{
2707	const struct rtw89_pci_info *info = rtwdev->pci_info;
2708	u32 dma_busy3 = info->dma_busy3_reg;
2709	u32 check, dma_busy;
2710	int ret;
2711
2712	check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2713
2714	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2715	10, 100, false, rtwdev, dma_busy3);
2716	if (ret)
2717	return ret;
2718
2719	return 0;
2720	}
2721
2722	static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2723	{
2724	u32 ret;
2725
2726	ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev);
2727	if (ret) {
2728	rtw89_err(rtwdev, "txdma ch busy\n");
2729	return ret;
2730	}
2731
2732	ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev);
2733	if (ret) {
2734	rtw89_err(rtwdev, "rxdma ch busy\n");
2735	return ret;
2736	}
2737
2738	return 0;
2739	}
2740
2741	static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2742	{
2743	const struct rtw89_pci_info *info = rtwdev->pci_info;
2744	enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
2745	enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
2746	enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
2747	enum mac_ax_tag_mode tag_mode = info->tag_mode;
2748	enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
2749	enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
2750	enum mac_ax_tx_burst tx_burst = info->tx_burst;
2751	enum mac_ax_rx_burst rx_burst = info->rx_burst;
2752	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2753	u8 cv = rtwdev->hal.cv;
2754	u32 val32;
2755
2756	if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2757	if (chip_id == RTL8852A && cv == CHIP_CBV)
2758	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2759	} else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2760	if (chip_id == RTL8852A || chip_id == RTL8852B)
2761	rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2762	}
2763
2764	if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
2765	if (chip_id == RTL8852A && cv == CHIP_CBV)
2766	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2767	} else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
2768	if (chip_id == RTL8852A || chip_id == RTL8852B)
2769	rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2770	}
2771
2772	if (rxbd_mode == MAC_AX_RXBD_PKT) {
2773	rtw89_write32_clr(rtwdev, addr: info->init_cfg_reg, bit: info->rxbd_mode_bit);
2774	} else if (rxbd_mode == MAC_AX_RXBD_SEP) {
2775	rtw89_write32_set(rtwdev, addr: info->init_cfg_reg, bit: info->rxbd_mode_bit);
2776
2777	if (chip_id == RTL8852A || chip_id == RTL8852B)
2778	rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2779	B_AX_PCIE_RX_APPLEN_MASK, data: 0);
2780	}
2781
2782	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2783	rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, data: tx_burst);
2784	rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, data: rx_burst);
2785	} else if (chip_id == RTL8852C) {
2786	rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, data: tx_burst);
2787	rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, data: rx_burst);
2788	}
2789
2790	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2791	if (tag_mode == MAC_AX_TAG_SGL) {
2792	val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2793	~B_AX_LATENCY_CONTROL;
2794	rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, data: val32);
2795	} else if (tag_mode == MAC_AX_TAG_MULTI) {
2796	val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2797	B_AX_LATENCY_CONTROL;
2798	rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, data: val32);
2799	}
2800	}
2801
2802	rtw89_write32_mask(rtwdev, addr: info->exp_ctrl_reg, mask: info->max_tag_num_mask,
2803	data: info->multi_tag_num);
2804
2805	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
2806	rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2807	data: wd_dma_idle_intvl);
2808	rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2809	data: wd_dma_act_intvl);
2810	} else if (chip_id == RTL8852C) {
2811	rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2812	data: wd_dma_idle_intvl);
2813	rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2814	data: wd_dma_act_intvl);
2815	}
2816
2817	if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2818	rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2819	B_AX_HOST_ADDR_INFO_8B_SEL);
2820	rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2821	} else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2822	rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2823	B_AX_HOST_ADDR_INFO_8B_SEL);
2824	rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2825	}
2826
2827	return 0;
2828	}
2829
2830	static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2831	{
2832	const struct rtw89_pci_info *info = rtwdev->pci_info;
2833
2834	rtw89_pci_power_wake(rtwdev, pwr_up: false);
2835
2836	if (rtwdev->chip->chip_id == RTL8852A) {
2837	/* ltr sw trigger */
2838	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2839	}
2840	info->ltr_set(rtwdev, false);
2841	rtw89_pci_ctrl_dma_all(rtwdev, enable: false);
2842	rtw89_pci_clr_idx_all(rtwdev);
2843
2844	return 0;
2845	}
2846
2847	static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
2848	{
2849	const struct rtw89_pci_info *info = rtwdev->pci_info;
2850	int ret;
2851
2852	rtw89_pci_ber(rtwdev);
2853	rtw89_pci_rxdma_prefth(rtwdev);
2854	rtw89_pci_l1off_pwroff(rtwdev);
2855	rtw89_pci_deglitch_setting(rtwdev);
2856	ret = rtw89_pci_l2_rxen_lat(rtwdev);
2857	if (ret) {
2858	rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2859	return ret;
2860	}
2861
2862	rtw89_pci_aphy_pwrcut(rtwdev);
2863	rtw89_pci_hci_ldo(rtwdev);
2864	rtw89_pci_dphy_delay(rtwdev);
2865
2866	ret = rtw89_pci_autok_x(rtwdev);
2867	if (ret) {
2868	rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2869	return ret;
2870	}
2871
2872	ret = rtw89_pci_auto_refclk_cal(rtwdev, autook_en: false);
2873	if (ret) {
2874	rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2875	return ret;
2876	}
2877
2878	rtw89_pci_power_wake_ax(rtwdev, pwr_up: true);
2879	rtw89_pci_autoload_hang(rtwdev);
2880	rtw89_pci_l12_vmain(rtwdev);
2881	rtw89_pci_gen2_force_ib(rtwdev);
2882	rtw89_pci_l1_ent_lat(rtwdev);
2883	rtw89_pci_wd_exit_l1(rtwdev);
2884	rtw89_pci_set_sic(rtwdev);
2885	rtw89_pci_set_lbc(rtwdev);
2886	rtw89_pci_set_io_rcy(rtwdev);
2887	rtw89_pci_set_dbg(rtwdev);
2888	rtw89_pci_set_keep_reg(rtwdev);
2889
2890	rtw89_write32_set(rtwdev, addr: info->dma_stop1.addr, B_AX_STOP_WPDMA);
2891
2892	/* stop DMA activities */
2893	rtw89_pci_ctrl_dma_all(rtwdev, enable: false);
2894
2895	ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2896	if (ret) {
2897	rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2898	return ret;
2899	}
2900
2901	rtw89_pci_clr_idx_all(rtwdev);
2902	rtw89_pci_mode_op(rtwdev);
2903
2904	/* fill TRX BD indexes */
2905	rtw89_pci_ops_reset(rtwdev);
2906
2907	ret = rtw89_pci_rst_bdram_ax(rtwdev);
2908	if (ret) {
2909	rtw89_warn(rtwdev, "reset bdram busy\n");
2910	return ret;
2911	}
2912
2913	/* disable all channels except to FW CMD channel to download firmware */
2914	rtw89_pci_ctrl_txdma_ch_ax(rtwdev, enable: false);
2915	rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, enable: true);
2916
2917	/* start DMA activities */
2918	rtw89_pci_ctrl_dma_all(rtwdev, enable: true);
2919
2920	return 0;
2921	}
2922
2923	static int rtw89_pci_ops_mac_pre_deinit_ax(struct rtw89_dev *rtwdev)
2924	{
2925	rtw89_pci_power_wake_ax(rtwdev, pwr_up: false);
2926
2927	return 0;
2928	}
2929
2930	int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2931	{
2932	u32 val;
2933
2934	if (!en)
2935	return 0;
2936
2937	val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2938	if (rtw89_pci_ltr_is_err_reg_val(val))
2939	return -EINVAL;
2940	val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2941	if (rtw89_pci_ltr_is_err_reg_val(val))
2942	return -EINVAL;
2943	val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2944	if (rtw89_pci_ltr_is_err_reg_val(val))
2945	return -EINVAL;
2946	val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2947	if (rtw89_pci_ltr_is_err_reg_val(val))
2948	return -EINVAL;
2949
2950	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2951	B_AX_LTR_WD_NOEMP_CHK);
2952	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2953	PCI_LTR_SPC_500US);
2954	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2955	PCI_LTR_IDLE_TIMER_3_2MS);
2956	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, data: 0x28);
2957	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, data: 0x28);
2958	rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, data: 0x90039003);
2959	rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, data: 0x880b880b);
2960
2961	return 0;
2962	}
2963	EXPORT_SYMBOL(rtw89_pci_ltr_set);
2964
2965	int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
2966	{
2967	u32 dec_ctrl;
2968	u32 val32;
2969
2970	val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2971	if (rtw89_pci_ltr_is_err_reg_val(val: val32))
2972	return -EINVAL;
2973	val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2974	if (rtw89_pci_ltr_is_err_reg_val(val: val32))
2975	return -EINVAL;
2976	dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
2977	if (rtw89_pci_ltr_is_err_reg_val(val: dec_ctrl))
2978	return -EINVAL;
2979	val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
2980	if (rtw89_pci_ltr_is_err_reg_val(val: val32))
2981	return -EINVAL;
2982	val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
2983	if (rtw89_pci_ltr_is_err_reg_val(val: val32))
2984	return -EINVAL;
2985
2986	if (!en) {
2987	dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
2988	dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
2989	B_AX_LTR_REQ_DRV;
2990	} else {
2991	dec_ctrl |= B_AX_LTR_HW_DEC_EN;
2992	}
2993
2994	dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
2995	dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
2996
2997	if (en)
2998	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
2999	B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
3000	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
3001	PCI_LTR_IDLE_TIMER_3_2MS);
3002	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, data: 0x28);
3003	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, data: 0x28);
3004	rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, data: dec_ctrl);
3005	rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, data: 0x90039003);
3006	rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, data: 0x880b880b);
3007
3008	return 0;
3009	}
3010	EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
3011
3012	static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
3013	{
3014	const struct rtw89_pci_info *info = rtwdev->pci_info;
3015	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3016	int ret;
3017
3018	ret = info->ltr_set(rtwdev, true);
3019	if (ret) {
3020	rtw89_err(rtwdev, "pci ltr set fail\n");
3021	return ret;
3022	}
3023	if (chip_id == RTL8852A) {
3024	/* ltr sw trigger */
3025	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
3026	}
3027	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3028	/* ADDR info 8-byte mode */
3029	rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
3030	B_AX_HOST_ADDR_INFO_8B_SEL);
3031	rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
3032	}
3033
3034	/* enable DMA for all queues */
3035	rtw89_pci_ctrl_txdma_ch_ax(rtwdev, enable: true);
3036
3037	/* Release PCI IO */
3038	rtw89_write32_clr(rtwdev, addr: info->dma_stop1.addr,
3039	B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
3040
3041	return 0;
3042	}
3043
3044	static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
3045	struct pci_dev *pdev)
3046	{
3047	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3048	int ret;
3049
3050	ret = pci_enable_device(dev: pdev);
3051	if (ret) {
3052	rtw89_err(rtwdev, "failed to enable pci device\n");
3053	return ret;
3054	}
3055
3056	pci_set_master(dev: pdev);
3057	pci_set_drvdata(pdev, data: rtwdev->hw);
3058
3059	rtwpci->pdev = pdev;
3060
3061	return 0;
3062	}
3063
3064	static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
3065	struct pci_dev *pdev)
3066	{
3067	pci_disable_device(dev: pdev);
3068	}
3069
3070	static bool rtw89_pci_chip_is_manual_dac(struct rtw89_dev *rtwdev)
3071	{
3072	const struct rtw89_chip_info *chip = rtwdev->chip;
3073
3074	switch (chip->chip_id) {
3075	case RTL8852A:
3076	case RTL8852B:
3077	case RTL8851B:
3078	case RTL8852BT:
3079	return true;
3080	default:
3081	return false;
3082	}
3083	}
3084
3085	static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev)
3086	{
3087	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3088	struct pci_dev *bridge = pci_upstream_bridge(dev: rtwpci->pdev);
3089
3090	if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3091	return true;
3092
3093	if (!bridge)
3094	return false;
3095
3096	switch (bridge->vendor) {
3097	case PCI_VENDOR_ID_INTEL:
3098	return true;
3099	case PCI_VENDOR_ID_ASMEDIA:
3100	if (bridge->device == 0x2806)
3101	return true;
3102	break;
3103	}
3104
3105	return false;
3106	}
3107
3108	static int rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev, bool force)
3109	{
3110	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3111	struct pci_dev *pdev = rtwpci->pdev;
3112	int ret;
3113	u8 val;
3114
3115	if (!rtwpci->enable_dac && !force)
3116	return 0;
3117
3118	if (!rtw89_pci_chip_is_manual_dac(rtwdev))
3119	return 0;
3120
3121	/* Configure DAC only via PCI config API, not DBI interfaces */
3122	ret = pci_read_config_byte(dev: pdev, RTW89_PCIE_L1_CTRL, val: &val);
3123	if (ret)
3124	return ret;
3125
3126	val |= RTW89_PCIE_BIT_EN_64BITS;
3127	return pci_write_config_byte(dev: pdev, RTW89_PCIE_L1_CTRL, val);
3128	}
3129
3130	static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
3131	struct pci_dev *pdev)
3132	{
3133	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3134	unsigned long resource_len;
3135	u8 bar_id = 2;
3136	int ret;
3137
3138	ret = pci_request_regions(pdev, KBUILD_MODNAME);
3139	if (ret) {
3140	rtw89_err(rtwdev, "failed to request pci regions\n");
3141	goto err;
3142	}
3143
3144	if (!rtw89_pci_is_dac_compatible_bridge(rtwdev))
3145	goto try_dac_done;
3146
3147	ret = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(36));
3148	if (!ret) {
3149	ret = rtw89_pci_cfg_dac(rtwdev, force: true);
3150	if (!ret) {
3151	rtwpci->enable_dac = true;
3152	goto try_dac_done;
3153	}
3154
3155	ret = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32));
3156	if (ret) {
3157	rtw89_err(rtwdev,
3158	"failed to set dma and consistent mask to 32/36-bit\n");
3159	goto err_release_regions;
3160	}
3161	}
3162	try_dac_done:
3163
3164	resource_len = pci_resource_len(pdev, bar_id);
3165	rtwpci->mmap = pci_iomap(dev: pdev, bar: bar_id, max: resource_len);
3166	if (!rtwpci->mmap) {
3167	rtw89_err(rtwdev, "failed to map pci io\n");
3168	ret = -EIO;
3169	goto err_release_regions;
3170	}
3171
3172	return 0;
3173
3174	err_release_regions:
3175	pci_release_regions(pdev);
3176	err:
3177	return ret;
3178	}
3179
3180	static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
3181	struct pci_dev *pdev)
3182	{
3183	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3184
3185	if (rtwpci->mmap) {
3186	pci_iounmap(dev: pdev, rtwpci->mmap);
3187	pci_release_regions(pdev);
3188	}
3189	}
3190
3191	static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
3192	struct pci_dev *pdev,
3193	struct rtw89_pci_tx_ring *tx_ring)
3194	{
3195	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
3196	u8 *head = wd_ring->head;
3197	dma_addr_t dma = wd_ring->dma;
3198	u32 page_size = wd_ring->page_size;
3199	u32 page_num = wd_ring->page_num;
3200	u32 ring_sz = page_size * page_num;
3201
3202	dma_free_coherent(dev: &pdev->dev, size: ring_sz, cpu_addr: head, dma_handle: dma);
3203	wd_ring->head = NULL;
3204	}
3205
3206	static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
3207	struct pci_dev *pdev,
3208	struct rtw89_pci_tx_ring *tx_ring)
3209	{
3210	int ring_sz;
3211	u8 *head;
3212	dma_addr_t dma;
3213
3214	head = tx_ring->bd_ring.head;
3215	dma = tx_ring->bd_ring.dma;
3216	ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len;
3217	dma_free_coherent(dev: &pdev->dev, size: ring_sz, cpu_addr: head, dma_handle: dma);
3218
3219	tx_ring->bd_ring.head = NULL;
3220	}
3221
3222	static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
3223	struct pci_dev *pdev)
3224	{
3225	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3226	const struct rtw89_pci_info *info = rtwdev->pci_info;
3227	struct rtw89_pci_tx_ring *tx_ring;
3228	int i;
3229
3230	for (i = 0; i < RTW89_TXCH_NUM; i++) {
3231	if (info->tx_dma_ch_mask & BIT(i))
3232	continue;
3233	tx_ring = &rtwpci->tx_rings[i];
3234	rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3235	rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3236	}
3237	}
3238
3239	static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
3240	struct pci_dev *pdev,
3241	struct rtw89_pci_rx_ring *rx_ring)
3242	{
3243	struct rtw89_pci_rx_info *rx_info;
3244	struct sk_buff *skb;
3245	dma_addr_t dma;
3246	u32 buf_sz;
3247	u8 *head;
3248	int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len;
3249	int i;
3250
3251	buf_sz = rx_ring->buf_sz;
3252	for (i = 0; i < rx_ring->bd_ring.len; i++) {
3253	skb = rx_ring->buf[i];
3254	if (!skb)
3255	continue;
3256
3257	rx_info = RTW89_PCI_RX_SKB_CB(skb);
3258	dma = rx_info->dma;
3259	dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3260	dev_kfree_skb(skb);
3261	rx_ring->buf[i] = NULL;
3262	}
3263
3264	head = rx_ring->bd_ring.head;
3265	dma = rx_ring->bd_ring.dma;
3266	dma_free_coherent(dev: &pdev->dev, size: ring_sz, cpu_addr: head, dma_handle: dma);
3267
3268	rx_ring->bd_ring.head = NULL;
3269	}
3270
3271	static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
3272	struct pci_dev *pdev)
3273	{
3274	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3275	struct rtw89_pci_rx_ring *rx_ring;
3276	int i;
3277
3278	for (i = 0; i < RTW89_RXCH_NUM; i++) {
3279	rx_ring = &rtwpci->rx_rings[i];
3280	rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3281	}
3282	}
3283
3284	static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
3285	struct pci_dev *pdev)
3286	{
3287	rtw89_pci_free_rx_rings(rtwdev, pdev);
3288	rtw89_pci_free_tx_rings(rtwdev, pdev);
3289	}
3290
3291	static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
3292	struct rtw89_pci_rx_ring *rx_ring,
3293	struct sk_buff *skb, int buf_sz, u32 idx)
3294	{
3295	struct rtw89_pci_rx_info *rx_info;
3296	struct rtw89_pci_rx_bd_32 *rx_bd;
3297	dma_addr_t dma;
3298
3299	if (!skb)
3300	return -EINVAL;
3301
3302	dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
3303	if (dma_mapping_error(dev: &pdev->dev, dma_addr: dma))
3304	return -EBUSY;
3305
3306	rx_info = RTW89_PCI_RX_SKB_CB(skb);
3307	rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
3308
3309	memset(rx_bd, 0, sizeof(*rx_bd));
3310	rx_bd->buf_size = cpu_to_le16(buf_sz);
3311	rx_bd->dma = cpu_to_le32(dma);
3312	rx_bd->opt = le16_encode_bits(upper_32_bits(dma), RTW89_PCI_RXBD_OPT_DMA_HI);
3313	rx_info->dma = dma;
3314
3315	return 0;
3316	}
3317
3318	static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
3319	struct pci_dev *pdev,
3320	struct rtw89_pci_tx_ring *tx_ring,
3321	enum rtw89_tx_channel txch)
3322	{
3323	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
3324	struct rtw89_pci_tx_wd *txwd;
3325	dma_addr_t dma;
3326	dma_addr_t cur_paddr;
3327	u8 *head;
3328	u8 *cur_vaddr;
3329	u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
3330	u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
3331	u32 ring_sz = page_size * page_num;
3332	u32 page_offset;
3333	int i;
3334
3335	/* FWCMD queue doesn't use txwd as pages */
3336	if (txch == RTW89_TXCH_CH12)
3337	return 0;
3338
3339	head = dma_alloc_coherent(dev: &pdev->dev, size: ring_sz, dma_handle: &dma, GFP_KERNEL);
3340	if (!head)
3341	return -ENOMEM;
3342
3343	INIT_LIST_HEAD(list: &wd_ring->free_pages);
3344	wd_ring->head = head;
3345	wd_ring->dma = dma;
3346	wd_ring->page_size = page_size;
3347	wd_ring->page_num = page_num;
3348
3349	page_offset = 0;
3350	for (i = 0; i < page_num; i++) {
3351	txwd = &wd_ring->pages[i];
3352	cur_paddr = dma + page_offset;
3353	cur_vaddr = head + page_offset;
3354
3355	skb_queue_head_init(list: &txwd->queue);
3356	INIT_LIST_HEAD(list: &txwd->list);
3357	txwd->paddr = cur_paddr;
3358	txwd->vaddr = cur_vaddr;
3359	txwd->len = page_size;
3360	txwd->seq = i;
3361	rtw89_pci_enqueue_txwd(tx_ring, txwd);
3362
3363	page_offset += page_size;
3364	}
3365
3366	return 0;
3367	}
3368
3369	static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
3370	struct pci_dev *pdev,
3371	struct rtw89_pci_tx_ring *tx_ring,
3372	u32 desc_size, u32 len,
3373	enum rtw89_tx_channel txch)
3374	{
3375	const struct rtw89_pci_ch_dma_addr *txch_addr;
3376	int ring_sz = desc_size * len;
3377	u8 *head;
3378	dma_addr_t dma;
3379	int ret;
3380
3381	ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
3382	if (ret) {
3383	rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
3384	goto err;
3385	}
3386
3387	ret = rtw89_pci_get_txch_addrs(rtwdev, txch, addr: &txch_addr);
3388	if (ret) {
3389	rtw89_err(rtwdev, "failed to get address of txch %d", txch);
3390	goto err_free_wd_ring;
3391	}
3392
3393	head = dma_alloc_coherent(dev: &pdev->dev, size: ring_sz, dma_handle: &dma, GFP_KERNEL);
3394	if (!head) {
3395	ret = -ENOMEM;
3396	goto err_free_wd_ring;
3397	}
3398
3399	INIT_LIST_HEAD(list: &tx_ring->busy_pages);
3400	tx_ring->bd_ring.head = head;
3401	tx_ring->bd_ring.dma = dma;
3402	tx_ring->bd_ring.len = len;
3403	tx_ring->bd_ring.desc_size = desc_size;
3404	tx_ring->bd_ring.addr = *txch_addr;
3405	tx_ring->bd_ring.wp = 0;
3406	tx_ring->bd_ring.rp = 0;
3407	tx_ring->txch = txch;
3408
3409	return 0;
3410
3411	err_free_wd_ring:
3412	rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3413	err:
3414	return ret;
3415	}
3416
3417	static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
3418	struct pci_dev *pdev)
3419	{
3420	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3421	const struct rtw89_pci_info *info = rtwdev->pci_info;
3422	struct rtw89_pci_tx_ring *tx_ring;
3423	u32 desc_size;
3424	u32 len;
3425	u32 i, tx_allocated;
3426	int ret;
3427
3428	for (i = 0; i < RTW89_TXCH_NUM; i++) {
3429	if (info->tx_dma_ch_mask & BIT(i))
3430	continue;
3431	tx_ring = &rtwpci->tx_rings[i];
3432	desc_size = sizeof(struct rtw89_pci_tx_bd_32);
3433	len = RTW89_PCI_TXBD_NUM_MAX;
3434	ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3435	desc_size, len, txch: i);
3436	if (ret) {
3437	rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3438	goto err_free;
3439	}
3440	}
3441
3442	return 0;
3443
3444	err_free:
3445	tx_allocated = i;
3446	for (i = 0; i < tx_allocated; i++) {
3447	tx_ring = &rtwpci->tx_rings[i];
3448	rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3449	}
3450
3451	return ret;
3452	}
3453
3454	static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3455	struct pci_dev *pdev,
3456	struct rtw89_pci_rx_ring *rx_ring,
3457	u32 desc_size, u32 len, u32 rxch)
3458	{
3459	const struct rtw89_pci_info *info = rtwdev->pci_info;
3460	const struct rtw89_pci_ch_dma_addr *rxch_addr;
3461	struct sk_buff *skb;
3462	u8 *head;
3463	dma_addr_t dma;
3464	int ring_sz = desc_size * len;
3465	int buf_sz = RTW89_PCI_RX_BUF_SIZE;
3466	int i, allocated;
3467	int ret;
3468
3469	ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, addr: &rxch_addr);
3470	if (ret) {
3471	rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3472	return ret;
3473	}
3474
3475	head = dma_alloc_coherent(dev: &pdev->dev, size: ring_sz, dma_handle: &dma, GFP_KERNEL);
3476	if (!head) {
3477	ret = -ENOMEM;
3478	goto err;
3479	}
3480
3481	rx_ring->bd_ring.head = head;
3482	rx_ring->bd_ring.dma = dma;
3483	rx_ring->bd_ring.len = len;
3484	rx_ring->bd_ring.desc_size = desc_size;
3485	rx_ring->bd_ring.addr = *rxch_addr;
3486	if (info->rx_ring_eq_is_full)
3487	rx_ring->bd_ring.wp = len - 1;
3488	else
3489	rx_ring->bd_ring.wp = 0;
3490	rx_ring->bd_ring.rp = 0;
3491	rx_ring->buf_sz = buf_sz;
3492	rx_ring->diliver_skb = NULL;
3493	rx_ring->diliver_desc.ready = false;
3494	rx_ring->target_rx_tag = 0;
3495
3496	for (i = 0; i < len; i++) {
3497	skb = dev_alloc_skb(length: buf_sz);
3498	if (!skb) {
3499	ret = -ENOMEM;
3500	goto err_free;
3501	}
3502
3503	memset(skb->data, 0, buf_sz);
3504	rx_ring->buf[i] = skb;
3505	ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3506	buf_sz, idx: i);
3507	if (ret) {
3508	rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3509	dev_kfree_skb_any(skb);
3510	rx_ring->buf[i] = NULL;
3511	goto err_free;
3512	}
3513	}
3514
3515	return 0;
3516
3517	err_free:
3518	allocated = i;
3519	for (i = 0; i < allocated; i++) {
3520	skb = rx_ring->buf[i];
3521	if (!skb)
3522	continue;
3523	dma = *((dma_addr_t *)skb->cb);
3524	dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3525	dev_kfree_skb(skb);
3526	rx_ring->buf[i] = NULL;
3527	}
3528
3529	head = rx_ring->bd_ring.head;
3530	dma = rx_ring->bd_ring.dma;
3531	dma_free_coherent(dev: &pdev->dev, size: ring_sz, cpu_addr: head, dma_handle: dma);
3532
3533	rx_ring->bd_ring.head = NULL;
3534	err:
3535	return ret;
3536	}
3537
3538	static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3539	struct pci_dev *pdev)
3540	{
3541	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3542	struct rtw89_pci_rx_ring *rx_ring;
3543	u32 desc_size;
3544	u32 len;
3545	int i, rx_allocated;
3546	int ret;
3547
3548	for (i = 0; i < RTW89_RXCH_NUM; i++) {
3549	rx_ring = &rtwpci->rx_rings[i];
3550	desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3551	len = RTW89_PCI_RXBD_NUM_MAX;
3552	ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3553	desc_size, len, rxch: i);
3554	if (ret) {
3555	rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3556	goto err_free;
3557	}
3558	}
3559
3560	return 0;
3561
3562	err_free:
3563	rx_allocated = i;
3564	for (i = 0; i < rx_allocated; i++) {
3565	rx_ring = &rtwpci->rx_rings[i];
3566	rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3567	}
3568
3569	return ret;
3570	}
3571
3572	static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3573	struct pci_dev *pdev)
3574	{
3575	int ret;
3576
3577	ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3578	if (ret) {
3579	rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3580	goto err;
3581	}
3582
3583	ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3584	if (ret) {
3585	rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3586	goto err_free_tx_rings;
3587	}
3588
3589	return 0;
3590
3591	err_free_tx_rings:
3592	rtw89_pci_free_tx_rings(rtwdev, pdev);
3593	err:
3594	return ret;
3595	}
3596
3597	static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3598	struct rtw89_pci *rtwpci)
3599	{
3600	skb_queue_head_init(list: &rtwpci->h2c_queue);
3601	skb_queue_head_init(list: &rtwpci->h2c_release_queue);
3602	}
3603
3604	static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3605	struct pci_dev *pdev)
3606	{
3607	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3608	int ret;
3609
3610	ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3611	if (ret) {
3612	rtw89_err(rtwdev, "failed to setup pci mapping\n");
3613	goto err;
3614	}
3615
3616	ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3617	if (ret) {
3618	rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3619	goto err_pci_unmap;
3620	}
3621
3622	rtw89_pci_h2c_init(rtwdev, rtwpci);
3623
3624	spin_lock_init(&rtwpci->irq_lock);
3625	spin_lock_init(&rtwpci->trx_lock);
3626
3627	return 0;
3628
3629	err_pci_unmap:
3630	rtw89_pci_clear_mapping(rtwdev, pdev);
3631	err:
3632	return ret;
3633	}
3634
3635	static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3636	struct pci_dev *pdev)
3637	{
3638	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3639
3640	rtw89_pci_free_trx_rings(rtwdev, pdev);
3641	rtw89_pci_clear_mapping(rtwdev, pdev);
3642	rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3643	cnt: skb_queue_len(list_: &rtwpci->h2c_queue), release_all: true);
3644	}
3645
3646	void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3647	{
3648	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3649	const struct rtw89_chip_info *chip = rtwdev->chip;
3650	u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN;
3651
3652	if (chip->chip_id == RTL8851B)
3653	hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND;
3654
3655	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3656
3657	if (rtwpci->under_recovery) {
3658	rtwpci->intrs[0] = hs0isr_ind_int_en;
3659	rtwpci->intrs[1] = 0;
3660	} else {
3661	rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3662	B_AX_RXDMA_INT_EN |
3663	B_AX_RXP1DMA_INT_EN |
3664	B_AX_RPQDMA_INT_EN |
3665	B_AX_RXDMA_STUCK_INT_EN |
3666	B_AX_RDU_INT_EN |
3667	B_AX_RPQBD_FULL_INT_EN |
3668	hs0isr_ind_int_en;
3669
3670	rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3671	}
3672	}
3673	EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3674
3675	static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3676	{
3677	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3678
3679	rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
3680	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3681	rtwpci->intrs[0] = 0;
3682	rtwpci->intrs[1] = 0;
3683	}
3684
3685	static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3686	{
3687	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3688
3689	rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
3690	B_AX_HS1ISR_IND_INT_EN |
3691	B_AX_HS0ISR_IND_INT_EN;
3692	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3693	rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3694	B_AX_RXDMA_INT_EN |
3695	B_AX_RXP1DMA_INT_EN |
3696	B_AX_RPQDMA_INT_EN |
3697	B_AX_RXDMA_STUCK_INT_EN |
3698	B_AX_RDU_INT_EN |
3699	B_AX_RPQBD_FULL_INT_EN;
3700	rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3701	}
3702
3703	static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3704	{
3705	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3706
3707	rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
3708	B_AX_HS0ISR_IND_INT_EN;
3709	rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3710	rtwpci->intrs[0] = 0;
3711	rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3712	}
3713
3714	void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3715	{
3716	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3717
3718	if (rtwpci->under_recovery)
3719	rtw89_pci_recovery_intr_mask_v1(rtwdev);
3720	else if (rtwpci->low_power)
3721	rtw89_pci_low_power_intr_mask_v1(rtwdev);
3722	else
3723	rtw89_pci_default_intr_mask_v1(rtwdev);
3724	}
3725	EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
3726
3727	static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
3728	{
3729	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3730
3731	rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
3732	rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3733	rtwpci->intrs[0] = 0;
3734	rtwpci->intrs[1] = 0;
3735	}
3736
3737	static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
3738	{
3739	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3740
3741	rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 |
3742	B_BE_HS0_IND_INT_EN0;
3743	rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3744	rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 |
3745	B_BE_RDU_CH0_INT_IMR_V1;
3746	rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3747	B_BE_PCIE_RX_RPQ0_IMR0_V1;
3748	}
3749
3750	static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
3751	{
3752	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3753
3754	rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 |
3755	B_BE_HS1_IND_INT_EN0;
3756	rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3757	rtwpci->intrs[0] = 0;
3758	rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3759	B_BE_PCIE_RX_RPQ0_IMR0_V1;
3760	}
3761
3762	void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
3763	{
3764	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3765
3766	if (rtwpci->under_recovery)
3767	rtw89_pci_recovery_intr_mask_v2(rtwdev);
3768	else if (rtwpci->low_power)
3769	rtw89_pci_low_power_intr_mask_v2(rtwdev);
3770	else
3771	rtw89_pci_default_intr_mask_v2(rtwdev);
3772	}
3773	EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2);
3774
3775	static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3776	struct pci_dev *pdev)
3777	{
3778	unsigned long flags = 0;
3779	int ret;
3780
3781	flags |= PCI_IRQ_INTX | PCI_IRQ_MSI;
3782	ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 1, flags);
3783	if (ret < 0) {
3784	rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3785	goto err;
3786	}
3787
3788	ret = devm_request_threaded_irq(dev: rtwdev->dev, irq: pdev->irq,
3789	handler: rtw89_pci_interrupt_handler,
3790	thread_fn: rtw89_pci_interrupt_threadfn,
3791	IRQF_SHARED, KBUILD_MODNAME, dev_id: rtwdev);
3792	if (ret) {
3793	rtw89_err(rtwdev, "failed to request threaded irq\n");
3794	goto err_free_vector;
3795	}
3796
3797	rtw89_chip_config_intr_mask(rtwdev, cfg: RTW89_PCI_INTR_MASK_RESET);
3798
3799	return 0;
3800
3801	err_free_vector:
3802	pci_free_irq_vectors(dev: pdev);
3803	err:
3804	return ret;
3805	}
3806
3807	static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3808	struct pci_dev *pdev)
3809	{
3810	devm_free_irq(dev: rtwdev->dev, irq: pdev->irq, dev_id: rtwdev);
3811	pci_free_irq_vectors(dev: pdev);
3812	}
3813
3814	static u16 gray_code_to_bin(u16 gray_code)
3815	{
3816	u16 binary = gray_code;
3817
3818	while (gray_code) {
3819	gray_code >>= 1;
3820	binary ^= gray_code;
3821	}
3822
3823	return binary;
3824	}
3825
3826	static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3827	{
3828	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3829	struct pci_dev *pdev = rtwpci->pdev;
3830	u16 val16, filter_out_val;
3831	u32 val, phy_offset;
3832	int ret;
3833
3834	if (rtwdev->chip->chip_id != RTL8852C)
3835	return 0;
3836
3837	val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3838	if (val == B_AX_ASPM_CTRL_L1)
3839	return 0;
3840
3841	ret = pci_read_config_dword(dev: pdev, RTW89_PCIE_L1_STS_V1, val: &val);
3842	if (ret)
3843	return ret;
3844
3845	val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
3846	if (val == RTW89_PCIE_GEN1_SPEED) {
3847	phy_offset = R_RAC_DIRECT_OFFSET_G1;
3848	} else if (val == RTW89_PCIE_GEN2_SPEED) {
3849	phy_offset = R_RAC_DIRECT_OFFSET_G2;
3850	val16 = rtw89_read16(rtwdev, addr: phy_offset + RAC_ANA10 * RAC_MULT);
3851	rtw89_write16_set(rtwdev, addr: phy_offset + RAC_ANA10 * RAC_MULT,
3852	bit: val16 | B_PCIE_BIT_PINOUT_DIS);
3853	rtw89_write16_set(rtwdev, addr: phy_offset + RAC_ANA19 * RAC_MULT,
3854	bit: val16 & ~B_PCIE_BIT_RD_SEL);
3855
3856	val16 = rtw89_read16_mask(rtwdev,
3857	addr: phy_offset + RAC_ANA1F * RAC_MULT,
3858	FILTER_OUT_EQ_MASK);
3859	val16 = gray_code_to_bin(gray_code: val16);
3860	filter_out_val = rtw89_read16(rtwdev, addr: phy_offset + RAC_ANA24 *
3861	RAC_MULT);
3862	filter_out_val &= ~REG_FILTER_OUT_MASK;
3863	filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
3864
3865	rtw89_write16(rtwdev, addr: phy_offset + RAC_ANA24 * RAC_MULT,
3866	data: filter_out_val);
3867	rtw89_write16_set(rtwdev, addr: phy_offset + RAC_ANA0A * RAC_MULT,
3868	B_BAC_EQ_SEL);
3869	rtw89_write16_set(rtwdev,
3870	R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
3871	B_PCIE_BIT_PSAVE);
3872	} else {
3873	return -EOPNOTSUPP;
3874	}
3875	rtw89_write16_set(rtwdev, addr: phy_offset + RAC_ANA0C * RAC_MULT,
3876	B_PCIE_BIT_PSAVE);
3877
3878	return 0;
3879	}
3880
3881	static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3882	{
3883	const struct rtw89_pci_info *info = rtwdev->pci_info;
3884	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3885
3886	if (rtw89_pci_disable_clkreq)
3887	return;
3888
3889	gen_def->clkreq_set(rtwdev, enable);
3890	}
3891
3892	static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable)
3893	{
3894	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3895	int ret;
3896
3897	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3898	data: PCIE_CLKDLY_HW_30US);
3899	if (ret)
3900	rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3901
3902	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3903	if (enable)
3904	ret = rtw89_pci_config_byte_set(rtwdev,
3905	RTW89_PCIE_L1_CTRL,
3906	RTW89_PCIE_BIT_CLK);
3907	else
3908	ret = rtw89_pci_config_byte_clr(rtwdev,
3909	RTW89_PCIE_L1_CTRL,
3910	RTW89_PCIE_BIT_CLK);
3911	if (ret)
3912	rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3913	enable ? "set" : "unset", ret);
3914	} else if (chip_id == RTL8852C) {
3915	rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3916	B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
3917	if (enable)
3918	rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3919	B_AX_CLK_REQ_N);
3920	else
3921	rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3922	B_AX_CLK_REQ_N);
3923	}
3924	}
3925
3926	static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3927	{
3928	const struct rtw89_pci_info *info = rtwdev->pci_info;
3929	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3930
3931	if (rtw89_pci_disable_aspm_l1)
3932	return;
3933
3934	gen_def->aspm_set(rtwdev, enable);
3935	}
3936
3937	static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable)
3938	{
3939	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3940	u8 value = 0;
3941	int ret;
3942
3943	ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value: &value);
3944	if (ret)
3945	rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
3946
3947	u8p_replace_bits(p: &value, val: PCIE_L1DLY_16US, RTW89_L1DLY_MASK);
3948	u8p_replace_bits(p: &value, val: PCIE_L0SDLY_4US, RTW89_L0DLY_MASK);
3949
3950	ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, data: value);
3951	if (ret)
3952	rtw89_warn(rtwdev, "failed to read ASPM Delay\n");
3953
3954	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
3955	if (enable)
3956	ret = rtw89_pci_config_byte_set(rtwdev,
3957	RTW89_PCIE_L1_CTRL,
3958	RTW89_PCIE_BIT_L1);
3959	else
3960	ret = rtw89_pci_config_byte_clr(rtwdev,
3961	RTW89_PCIE_L1_CTRL,
3962	RTW89_PCIE_BIT_L1);
3963	} else if (chip_id == RTL8852C) {
3964	if (enable)
3965	rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3966	B_AX_ASPM_CTRL_L1);
3967	else
3968	rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3969	B_AX_ASPM_CTRL_L1);
3970	}
3971	if (ret)
3972	rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3973	enable ? "set" : "unset", ret);
3974	}
3975
3976	static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3977	{
3978	enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
3979	const struct rtw89_pci_info *info = rtwdev->pci_info;
3980	struct rtw89_traffic_stats *stats = &rtwdev->stats;
3981	enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
3982	enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
3983	u32 val = 0;
3984
3985	if (rtwdev->scanning ||
3986	(tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH))
3987	goto out;
3988
3989	if (chip_gen == RTW89_CHIP_BE)
3990	val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN;
3991	else
3992	val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
3993	FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
3994	FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
3995	FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
3996
3997	out:
3998	rtw89_write32(rtwdev, addr: info->mit_addr, data: val);
3999	}
4000
4001	static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
4002	{
4003	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4004	struct pci_dev *pdev = rtwpci->pdev;
4005	u16 link_ctrl;
4006	int ret;
4007
4008	/* Though there is standard PCIE configuration space to set the
4009	* link control register, but by Realtek's design, driver should
4010	* check if host supports CLKREQ/ASPM to enable the HW module.
4011	*
4012	* These functions are implemented by two HW modules associated,
4013	* one is responsible to access PCIE configuration space to
4014	* follow the host settings, and another is in charge of doing
4015	* CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
4016	* the host does not support it, and due to some reasons or wrong
4017	* settings (ex. CLKREQ# not Bi-Direction), it could lead to device
4018	* loss if HW misbehaves on the link.
4019	*
4020	* Hence it's designed that driver should first check the PCIE
4021	* configuration space is sync'ed and enabled, then driver can turn
4022	* on the other module that is actually working on the mechanism.
4023	*/
4024	ret = pcie_capability_read_word(dev: pdev, PCI_EXP_LNKCTL, val: &link_ctrl);
4025	if (ret) {
4026	rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
4027	return;
4028	}
4029
4030	if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
4031	rtw89_pci_clkreq_set(rtwdev, enable: true);
4032
4033	if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
4034	rtw89_pci_aspm_set(rtwdev, enable: true);
4035	}
4036
4037	static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
4038	{
4039	const struct rtw89_pci_info *info = rtwdev->pci_info;
4040	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4041
4042	if (rtw89_pci_disable_l1ss)
4043	return;
4044
4045	gen_def->l1ss_set(rtwdev, enable);
4046	}
4047
4048	static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable)
4049	{
4050	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4051	int ret;
4052
4053	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4054	if (enable)
4055	ret = rtw89_pci_config_byte_set(rtwdev,
4056	RTW89_PCIE_TIMER_CTRL,
4057	RTW89_PCIE_BIT_L1SUB);
4058	else
4059	ret = rtw89_pci_config_byte_clr(rtwdev,
4060	RTW89_PCIE_TIMER_CTRL,
4061	RTW89_PCIE_BIT_L1SUB);
4062	if (ret)
4063	rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
4064	enable ? "set" : "unset", ret);
4065	} else if (chip_id == RTL8852C) {
4066	ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
4067	RTW89_PCIE_BIT_ASPM_L11 |
4068	RTW89_PCIE_BIT_PCI_L11);
4069	if (ret)
4070	rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
4071	if (enable)
4072	rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4073	B_AX_L1SUB_DISABLE);
4074	else
4075	rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
4076	B_AX_L1SUB_DISABLE);
4077	}
4078	}
4079
4080	static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
4081	{
4082	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4083	struct pci_dev *pdev = rtwpci->pdev;
4084	u32 l1ss_cap_ptr, l1ss_ctrl;
4085
4086	if (rtw89_pci_disable_l1ss)
4087	return;
4088
4089	l1ss_cap_ptr = pci_find_ext_capability(dev: pdev, PCI_EXT_CAP_ID_L1SS);
4090	if (!l1ss_cap_ptr)
4091	return;
4092
4093	pci_read_config_dword(dev: pdev, where: l1ss_cap_ptr + PCI_L1SS_CTL1, val: &l1ss_ctrl);
4094
4095	if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
4096	rtw89_pci_l1ss_set(rtwdev, enable: true);
4097	}
4098
4099	static void rtw89_pci_cpl_timeout_cfg(struct rtw89_dev *rtwdev)
4100	{
4101	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4102	struct pci_dev *pdev = rtwpci->pdev;
4103
4104	pcie_capability_set_word(dev: pdev, PCI_EXP_DEVCTL2,
4105	PCI_EXP_DEVCTL2_COMP_TMOUT_DIS);
4106	}
4107
4108	static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
4109	{
4110	int ret = 0;
4111	u32 sts;
4112	u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
4113
4114	ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
4115	10, 1000, false, rtwdev,
4116	R_AX_PCIE_DMA_BUSY1);
4117	if (ret) {
4118	rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
4119	rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
4120	return -EINVAL;
4121	}
4122	return ret;
4123	}
4124
4125	static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
4126	{
4127	u32 val;
4128	int ret;
4129
4130	if (rtwdev->chip->chip_id == RTL8852C)
4131	return 0;
4132
4133	rtw89_pci_ctrl_dma_all(rtwdev, enable: false);
4134	ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4135	if (ret) {
4136	val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4137	rtw89_debug(rtwdev, mask: RTW89_DBG_HCI,
4138	fmt: "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
4139	R_AX_DBG_ERR_FLAG, val);
4140	if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
4141	rtw89_mac_ctrl_hci_dma_tx(rtwdev, enable: false);
4142	if (val & B_AX_RX_STUCK)
4143	rtw89_mac_ctrl_hci_dma_rx(rtwdev, enable: false);
4144	rtw89_mac_ctrl_hci_dma_trx(rtwdev, enable: true);
4145	ret = rtw89_pci_poll_io_idle_ax(rtwdev);
4146	val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
4147	rtw89_debug(rtwdev, mask: RTW89_DBG_HCI,
4148	fmt: "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
4149	R_AX_DBG_ERR_FLAG, val);
4150	}
4151
4152	return ret;
4153	}
4154
4155	static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
4156	{
4157	u32 ret;
4158
4159	if (rtwdev->chip->chip_id == RTL8852C)
4160	return 0;
4161
4162	rtw89_mac_ctrl_hci_dma_trx(rtwdev, enable: false);
4163	rtw89_mac_ctrl_hci_dma_trx(rtwdev, enable: true);
4164	rtw89_pci_clr_idx_all(rtwdev);
4165
4166	ret = rtw89_pci_rst_bdram_ax(rtwdev);
4167	if (ret)
4168	return ret;
4169
4170	rtw89_pci_ctrl_dma_all(rtwdev, enable: true);
4171	return ret;
4172	}
4173
4174	static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
4175	enum rtw89_lv1_rcvy_step step)
4176	{
4177	const struct rtw89_pci_info *info = rtwdev->pci_info;
4178	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4179	int ret;
4180
4181	switch (step) {
4182	case RTW89_LV1_RCVY_STEP_1:
4183	ret = gen_def->lv1rst_stop_dma(rtwdev);
4184	if (ret)
4185	rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
4186
4187	break;
4188
4189	case RTW89_LV1_RCVY_STEP_2:
4190	ret = gen_def->lv1rst_start_dma(rtwdev);
4191	if (ret)
4192	rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
4193	break;
4194
4195	default:
4196	return -EINVAL;
4197	}
4198
4199	return ret;
4200	}
4201
4202	static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
4203	{
4204	if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
4205	return;
4206
4207	if (rtwdev->chip->chip_id == RTL8852C) {
4208	rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4209	rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1));
4210	rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4211	rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1));
4212	} else {
4213	rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
4214	rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
4215	rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
4216	rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
4217	rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
4218	rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
4219	}
4220	}
4221
4222	static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
4223	{
4224	struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
4225	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
4226	const struct rtw89_pci_info *info = rtwdev->pci_info;
4227	const struct rtw89_pci_gen_def *gen_def = info->gen_def;
4228	unsigned long flags;
4229	int work_done;
4230
4231	rtwdev->napi_budget_countdown = budget;
4232
4233	rtw89_write32(rtwdev, addr: gen_def->isr_clear_rpq.addr, data: gen_def->isr_clear_rpq.data);
4234	work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget: rtwdev->napi_budget_countdown);
4235	if (work_done == budget)
4236	return budget;
4237
4238	rtw89_write32(rtwdev, addr: gen_def->isr_clear_rxq.addr, data: gen_def->isr_clear_rxq.data);
4239	work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget: rtwdev->napi_budget_countdown);
4240	if (work_done < budget && napi_complete_done(n: napi, work_done)) {
4241	spin_lock_irqsave(&rtwpci->irq_lock, flags);
4242	if (likely(rtwpci->running))
4243	rtw89_chip_enable_intr(rtwdev, rtwpci);
4244	spin_unlock_irqrestore(lock: &rtwpci->irq_lock, flags);
4245	}
4246
4247	return work_done;
4248	}
4249
4250	static
4251	void rtw89_check_pci_ssid_quirks(struct rtw89_dev *rtwdev,
4252	struct pci_dev *pdev,
4253	const struct rtw89_pci_ssid_quirk *ssid_quirks)
4254	{
4255	int i;
4256
4257	if (!ssid_quirks)
4258	return;
4259
4260	for (i = 0; i < 200; i++, ssid_quirks++) {
4261	if (ssid_quirks->vendor == 0 && ssid_quirks->device == 0)
4262	break;
4263
4264	if (ssid_quirks->vendor != pdev->vendor ||
4265	ssid_quirks->device != pdev->device ||
4266	ssid_quirks->subsystem_vendor != pdev->subsystem_vendor ||
4267	ssid_quirks->subsystem_device != pdev->subsystem_device)
4268	continue;
4269
4270	bitmap_or(dst: rtwdev->quirks, src1: rtwdev->quirks, src2: &ssid_quirks->bitmap,
4271	nbits: NUM_OF_RTW89_QUIRKS);
4272	rtwdev->custid = ssid_quirks->custid;
4273	break;
4274	}
4275
4276	rtw89_debug(rtwdev, mask: RTW89_DBG_HCI, fmt: "quirks=%*ph custid=%d\n",
4277	(int)sizeof(rtwdev->quirks), rtwdev->quirks, rtwdev->custid);
4278	}
4279
4280	static int __maybe_unused rtw89_pci_suspend(struct device *dev)
4281	{
4282	struct ieee80211_hw *hw = dev_get_drvdata(dev);
4283	struct rtw89_dev *rtwdev = hw->priv;
4284	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4285
4286	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4287	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4288	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4289	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4290	rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
4291	B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
4292	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
4293	B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
4294	} else {
4295	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4296	B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
4297	}
4298
4299	return 0;
4300	}
4301
4302	static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
4303	{
4304	if (rtwdev->chip->chip_id == RTL8852C)
4305	return;
4306
4307	/* Hardware need write the reg twice to ensure the setting work */
4308	rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4309	RTW89_PCIE_BIT_CFG_RST_MSTATE);
4310	rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
4311	RTW89_PCIE_BIT_CFG_RST_MSTATE);
4312	}
4313
4314	void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume)
4315	{
4316	if (resume)
4317	rtw89_pci_cfg_dac(rtwdev, force: false);
4318
4319	rtw89_pci_disable_eq(rtwdev);
4320	rtw89_pci_filter_out(rtwdev);
4321	rtw89_pci_cpl_timeout_cfg(rtwdev);
4322	rtw89_pci_link_cfg(rtwdev);
4323	rtw89_pci_l1ss_cfg(rtwdev);
4324	}
4325
4326	static int __maybe_unused rtw89_pci_resume(struct device *dev)
4327	{
4328	struct ieee80211_hw *hw = dev_get_drvdata(dev);
4329	struct rtw89_dev *rtwdev = hw->priv;
4330	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
4331
4332	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4333	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
4334	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
4335	if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) {
4336	rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
4337	B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
4338	rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
4339	B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
4340	} else {
4341	rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4342	B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
4343	rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
4344	B_AX_SEL_REQ_ENTR_L1);
4345	}
4346	rtw89_pci_l2_hci_ldo(rtwdev);
4347
4348	rtw89_pci_basic_cfg(rtwdev, resume: true);
4349
4350	return 0;
4351	}
4352
4353	SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
4354	EXPORT_SYMBOL(rtw89_pm_ops);
4355
4356	const struct rtw89_pci_gen_def rtw89_pci_gen_ax = {
4357	.isr_rdu = B_AX_RDU_INT,
4358	.isr_halt_c2h = B_AX_HALT_C2H_INT_EN,
4359	.isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN,
4360	.isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT},
4361	.isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT |
4362	B_AX_RDU_INT},
4363
4364	.mac_pre_init = rtw89_pci_ops_mac_pre_init_ax,
4365	.mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit_ax,
4366	.mac_post_init = rtw89_pci_ops_mac_post_init_ax,
4367
4368	.clr_idx_all = rtw89_pci_clr_idx_all_ax,
4369	.rst_bdram = rtw89_pci_rst_bdram_ax,
4370
4371	.lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax,
4372	.lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax,
4373
4374	.ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_ax,
4375	.ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_ax,
4376	.poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle_ax,
4377
4378	.aspm_set = rtw89_pci_aspm_set_ax,
4379	.clkreq_set = rtw89_pci_clkreq_set_ax,
4380	.l1ss_set = rtw89_pci_l1ss_set_ax,
4381
4382	.disable_eq = rtw89_pci_disable_eq_ax,
4383	.power_wake = rtw89_pci_power_wake_ax,
4384	};
4385	EXPORT_SYMBOL(rtw89_pci_gen_ax);
4386
4387	static const struct rtw89_hci_ops rtw89_pci_ops = {
4388	.tx_write = rtw89_pci_ops_tx_write,
4389	.tx_kick_off = rtw89_pci_ops_tx_kick_off,
4390	.flush_queues = rtw89_pci_ops_flush_queues,
4391	.reset = rtw89_pci_ops_reset,
4392	.start = rtw89_pci_ops_start,
4393	.stop = rtw89_pci_ops_stop,
4394	.pause = rtw89_pci_ops_pause,
4395	.switch_mode = rtw89_pci_ops_switch_mode,
4396	.recalc_int_mit = rtw89_pci_recalc_int_mit,
4397
4398	.read8 = rtw89_pci_ops_read8,
4399	.read16 = rtw89_pci_ops_read16,
4400	.read32 = rtw89_pci_ops_read32,
4401	.write8 = rtw89_pci_ops_write8,
4402	.write16 = rtw89_pci_ops_write16,
4403	.write32 = rtw89_pci_ops_write32,
4404
4405	.mac_pre_init = rtw89_pci_ops_mac_pre_init,
4406	.mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit,
4407	.mac_post_init = rtw89_pci_ops_mac_post_init,
4408	.deinit = rtw89_pci_ops_deinit,
4409
4410	.check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
4411	.mac_lv1_rcvy = rtw89_pci_ops_mac_lv1_recovery,
4412	.dump_err_status = rtw89_pci_ops_dump_err_status,
4413	.napi_poll = rtw89_pci_napi_poll,
4414
4415	.recovery_start = rtw89_pci_ops_recovery_start,
4416	.recovery_complete = rtw89_pci_ops_recovery_complete,
4417
4418	.ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch,
4419	.ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch,
4420	.ctrl_trxhci = rtw89_pci_ctrl_dma_trx,
4421	.poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle,
4422
4423	.clr_idx_all = rtw89_pci_clr_idx_all,
4424	.clear = rtw89_pci_clear_resource,
4425	.disable_intr = rtw89_pci_disable_intr_lock,
4426	.enable_intr = rtw89_pci_enable_intr_lock,
4427	.rst_bdram = rtw89_pci_reset_bdram,
4428	};
4429
4430	int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4431	{
4432	struct rtw89_dev *rtwdev;
4433	const struct rtw89_driver_info *info;
4434	const struct rtw89_pci_info *pci_info;
4435	int ret;
4436
4437	info = (const struct rtw89_driver_info *)id->driver_data;
4438
4439	rtwdev = rtw89_alloc_ieee80211_hw(device: &pdev->dev,
4440	bus_data_size: sizeof(struct rtw89_pci),
4441	chip: info->chip, variant: info->variant);
4442	if (!rtwdev) {
4443	dev_err(&pdev->dev, "failed to allocate hw\n");
4444	return -ENOMEM;
4445	}
4446
4447	pci_info = info->bus.pci;
4448
4449	rtwdev->pci_info = info->bus.pci;
4450	rtwdev->hci.ops = &rtw89_pci_ops;
4451	rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
4452	rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
4453	rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
4454
4455	rtw89_check_quirks(rtwdev, quirks: info->quirks);
4456	rtw89_check_pci_ssid_quirks(rtwdev, pdev, ssid_quirks: pci_info->ssid_quirks);
4457
4458	SET_IEEE80211_DEV(hw: rtwdev->hw, dev: &pdev->dev);
4459
4460	ret = rtw89_core_init(rtwdev);
4461	if (ret) {
4462	rtw89_err(rtwdev, "failed to initialise core\n");
4463	goto err_release_hw;
4464	}
4465
4466	ret = rtw89_pci_claim_device(rtwdev, pdev);
4467	if (ret) {
4468	rtw89_err(rtwdev, "failed to claim pci device\n");
4469	goto err_core_deinit;
4470	}
4471
4472	ret = rtw89_pci_setup_resource(rtwdev, pdev);
4473	if (ret) {
4474	rtw89_err(rtwdev, "failed to setup pci resource\n");
4475	goto err_declaim_pci;
4476	}
4477
4478	ret = rtw89_chip_info_setup(rtwdev);
4479	if (ret) {
4480	rtw89_err(rtwdev, "failed to setup chip information\n");
4481	goto err_clear_resource;
4482	}
4483
4484	rtw89_pci_basic_cfg(rtwdev, resume: false);
4485
4486	ret = rtw89_core_napi_init(rtwdev);
4487	if (ret) {
4488	rtw89_err(rtwdev, "failed to init napi\n");
4489	goto err_clear_resource;
4490	}
4491
4492	ret = rtw89_pci_request_irq(rtwdev, pdev);
4493	if (ret) {
4494	rtw89_err(rtwdev, "failed to request pci irq\n");
4495	goto err_deinit_napi;
4496	}
4497
4498	ret = rtw89_core_register(rtwdev);
4499	if (ret) {
4500	rtw89_err(rtwdev, "failed to register core\n");
4501	goto err_free_irq;
4502	}
4503
4504	set_bit(nr: RTW89_FLAG_PROBE_DONE, addr: rtwdev->flags);
4505
4506	return 0;
4507
4508	err_free_irq:
4509	rtw89_pci_free_irq(rtwdev, pdev);
4510	err_deinit_napi:
4511	rtw89_core_napi_deinit(rtwdev);
4512	err_clear_resource:
4513	rtw89_pci_clear_resource(rtwdev, pdev);
4514	err_declaim_pci:
4515	rtw89_pci_declaim_device(rtwdev, pdev);
4516	err_core_deinit:
4517	rtw89_core_deinit(rtwdev);
4518	err_release_hw:
4519	rtw89_free_ieee80211_hw(rtwdev);
4520
4521	return ret;
4522	}
4523	EXPORT_SYMBOL(rtw89_pci_probe);
4524
4525	void rtw89_pci_remove(struct pci_dev *pdev)
4526	{
4527	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
4528	struct rtw89_dev *rtwdev;
4529
4530	rtwdev = hw->priv;
4531
4532	rtw89_pci_free_irq(rtwdev, pdev);
4533	rtw89_core_napi_deinit(rtwdev);
4534	rtw89_core_unregister(rtwdev);
4535	rtw89_pci_clear_resource(rtwdev, pdev);
4536	rtw89_pci_declaim_device(rtwdev, pdev);
4537	rtw89_core_deinit(rtwdev);
4538	rtw89_free_ieee80211_hw(rtwdev);
4539	}
4540	EXPORT_SYMBOL(rtw89_pci_remove);
4541
4542	MODULE_AUTHOR("Realtek Corporation");
4543	MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver");
4544	MODULE_LICENSE("Dual BSD/GPL");
4545