1// SPDX-License-Identifier: BSD-3-Clause-Clear
2/*
3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
4 */
5
6#include <linux/dma-mapping.h>
7#include "mt76.h"
8#include "dma.h"
9
10static struct mt76_txwi_cache *
11mt76_alloc_txwi(struct mt76_dev *dev)
12{
13 struct mt76_txwi_cache *t;
14 dma_addr_t addr;
15 u8 *txwi;
16 int size;
17
18 size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
19 txwi = kzalloc(size, GFP_ATOMIC);
20 if (!txwi)
21 return NULL;
22
23 addr = dma_map_single(dev->dma_dev, txwi, dev->drv->txwi_size,
24 DMA_TO_DEVICE);
25 if (unlikely(dma_mapping_error(dev->dma_dev, addr))) {
26 kfree(objp: txwi);
27 return NULL;
28 }
29
30 t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
31 t->dma_addr = addr;
32
33 return t;
34}
35
36static struct mt76_txwi_cache *
37mt76_alloc_rxwi(struct mt76_dev *dev)
38{
39 struct mt76_txwi_cache *t;
40
41 t = kzalloc(L1_CACHE_ALIGN(sizeof(*t)), GFP_ATOMIC);
42 if (!t)
43 return NULL;
44
45 t->ptr = NULL;
46 return t;
47}
48
49static struct mt76_txwi_cache *
50__mt76_get_txwi(struct mt76_dev *dev)
51{
52 struct mt76_txwi_cache *t = NULL;
53
54 spin_lock(lock: &dev->lock);
55 if (!list_empty(head: &dev->txwi_cache)) {
56 t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
57 list);
58 list_del(entry: &t->list);
59 }
60 spin_unlock(lock: &dev->lock);
61
62 return t;
63}
64
65static struct mt76_txwi_cache *
66__mt76_get_rxwi(struct mt76_dev *dev)
67{
68 struct mt76_txwi_cache *t = NULL;
69
70 spin_lock_bh(lock: &dev->wed_lock);
71 if (!list_empty(head: &dev->rxwi_cache)) {
72 t = list_first_entry(&dev->rxwi_cache, struct mt76_txwi_cache,
73 list);
74 list_del(entry: &t->list);
75 }
76 spin_unlock_bh(lock: &dev->wed_lock);
77
78 return t;
79}
80
81static struct mt76_txwi_cache *
82mt76_get_txwi(struct mt76_dev *dev)
83{
84 struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
85
86 if (t)
87 return t;
88
89 return mt76_alloc_txwi(dev);
90}
91
92struct mt76_txwi_cache *
93mt76_get_rxwi(struct mt76_dev *dev)
94{
95 struct mt76_txwi_cache *t = __mt76_get_rxwi(dev);
96
97 if (t)
98 return t;
99
100 return mt76_alloc_rxwi(dev);
101}
102EXPORT_SYMBOL_GPL(mt76_get_rxwi);
103
104void
105mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
106{
107 if (!t)
108 return;
109
110 spin_lock(lock: &dev->lock);
111 list_add(new: &t->list, head: &dev->txwi_cache);
112 spin_unlock(lock: &dev->lock);
113}
114EXPORT_SYMBOL_GPL(mt76_put_txwi);
115
116void
117mt76_put_rxwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
118{
119 if (!t)
120 return;
121
122 spin_lock_bh(lock: &dev->wed_lock);
123 list_add(new: &t->list, head: &dev->rxwi_cache);
124 spin_unlock_bh(lock: &dev->wed_lock);
125}
126EXPORT_SYMBOL_GPL(mt76_put_rxwi);
127
128static void
129mt76_free_pending_txwi(struct mt76_dev *dev)
130{
131 struct mt76_txwi_cache *t;
132
133 local_bh_disable();
134 while ((t = __mt76_get_txwi(dev)) != NULL) {
135 dma_unmap_single(dev->dma_dev, t->dma_addr, dev->drv->txwi_size,
136 DMA_TO_DEVICE);
137 kfree(objp: mt76_get_txwi_ptr(dev, t));
138 }
139 local_bh_enable();
140}
141
142void
143mt76_free_pending_rxwi(struct mt76_dev *dev)
144{
145 struct mt76_txwi_cache *t;
146
147 local_bh_disable();
148 while ((t = __mt76_get_rxwi(dev)) != NULL) {
149 if (t->ptr)
150 mt76_put_page_pool_buf(buf: t->ptr, allow_direct: false);
151 kfree(objp: t);
152 }
153 local_bh_enable();
154}
155EXPORT_SYMBOL_GPL(mt76_free_pending_rxwi);
156
157static void
158mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q)
159{
160 if (!mt76_queue_is_wed_rro(q))
161 return;
162
163 q->magic_cnt = 0;
164 if (mt76_queue_is_wed_rro_ind(q)) {
165 struct mt76_wed_rro_desc *rro_desc;
166 u32 data1 = FIELD_PREP(RRO_IND_DATA1_MAGIC_CNT_MASK,
167 MT_DMA_WED_IND_CMD_CNT - 1);
168 int i;
169
170 rro_desc = (struct mt76_wed_rro_desc *)q->desc;
171 for (i = 0; i < q->ndesc; i++) {
172 struct mt76_wed_rro_ind *cmd;
173
174 cmd = (struct mt76_wed_rro_ind *)&rro_desc[i];
175 cmd->data1 = cpu_to_le32(data1);
176 }
177 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) {
178 struct mt76_rro_rxdmad_c *dmad = (void *)q->desc;
179 u32 data3 = FIELD_PREP(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK,
180 MT_DMA_MAGIC_CNT - 1);
181 int i;
182
183 for (i = 0; i < q->ndesc; i++)
184 dmad[i].data3 = cpu_to_le32(data3);
185 }
186}
187
188static void
189mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
190{
191 Q_WRITE(q, desc_base, q->desc_dma);
192 if ((q->flags & MT_QFLAG_WED_RRO_EN) && !mt76_npu_device_active(dev))
193 Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc);
194 else
195 Q_WRITE(q, ring_size, q->ndesc);
196
197 if (mt76_queue_is_npu_tx(q)) {
198 writel(val: q->desc_dma, addr: &q->regs->desc_base);
199 writel(val: q->ndesc, addr: &q->regs->ring_size);
200 }
201 q->head = Q_READ(q, dma_idx);
202 q->tail = q->head;
203}
204
205void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q,
206 bool reset_idx)
207{
208 if (!q || !q->ndesc)
209 return;
210
211 if (!mt76_queue_is_wed_rro_ind(q) &&
212 !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) {
213 int i;
214
215 /* clear descriptors */
216 for (i = 0; i < q->ndesc; i++)
217 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
218 }
219
220 mt76_dma_queue_magic_cnt_init(dev, q);
221 if (reset_idx) {
222 if (mt76_queue_is_emi(q))
223 *q->emi_cpu_idx = 0;
224 else
225 Q_WRITE(q, cpu_idx, 0);
226 Q_WRITE(q, dma_idx, 0);
227 }
228 mt76_dma_sync_idx(dev, q);
229}
230
231static int
232mt76_dma_add_rx_buf(struct mt76_dev *dev, struct mt76_queue *q,
233 struct mt76_queue_buf *buf, void *data)
234{
235 struct mt76_queue_entry *entry = &q->entry[q->head];
236 struct mt76_txwi_cache *txwi = NULL;
237 u32 buf1 = 0, ctrl, info = 0;
238 struct mt76_desc *desc;
239 int idx = q->head;
240 int rx_token;
241
242 if (mt76_queue_is_wed_rro_ind(q)) {
243 struct mt76_wed_rro_desc *rro_desc;
244
245 rro_desc = (struct mt76_wed_rro_desc *)q->desc;
246 data = &rro_desc[q->head];
247 goto done;
248 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) {
249 data = &q->desc[q->head];
250 goto done;
251 }
252
253 desc = &q->desc[q->head];
254 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
255#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
256 buf1 = FIELD_PREP(MT_DMA_CTL_SDP0_H, buf->addr >> 32);
257#endif
258
259 if (mt76_queue_is_wed_rx(q) || mt76_queue_is_wed_rro_data(q)) {
260 txwi = mt76_get_rxwi(dev);
261 if (!txwi)
262 return -ENOMEM;
263
264 rx_token = mt76_rx_token_consume(dev, ptr: data, r: txwi, phys: buf->addr);
265 if (rx_token < 0) {
266 mt76_put_rxwi(dev, txwi);
267 return -ENOMEM;
268 }
269
270 buf1 |= FIELD_PREP(MT_DMA_CTL_TOKEN, rx_token);
271 ctrl |= MT_DMA_CTL_TO_HOST;
272
273 txwi->qid = q - dev->q_rx;
274 }
275
276 if (mt76_queue_is_wed_rro_msdu_pg(q) &&
277 dev->drv->rx_rro_add_msdu_page) {
278 if (dev->drv->rx_rro_add_msdu_page(dev, q, buf->addr, data))
279 return -ENOMEM;
280 }
281
282 if (q->flags & MT_QFLAG_WED_RRO_EN) {
283 info |= FIELD_PREP(MT_DMA_MAGIC_MASK, q->magic_cnt);
284 if ((q->head + 1) == q->ndesc)
285 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT;
286 }
287
288 WRITE_ONCE(desc->buf0, cpu_to_le32(buf->addr));
289 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
290 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
291 WRITE_ONCE(desc->info, cpu_to_le32(info));
292
293done:
294 entry->dma_addr[0] = buf->addr;
295 entry->dma_len[0] = buf->len;
296 entry->txwi = txwi;
297 entry->buf = data;
298 entry->wcid = 0xffff;
299 entry->skip_buf1 = true;
300 q->head = (q->head + 1) % q->ndesc;
301 q->queued++;
302
303 return idx;
304}
305
306static int
307mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
308 struct mt76_queue_buf *buf, int nbufs, u32 info,
309 struct sk_buff *skb, void *txwi)
310{
311 struct mt76_queue_entry *entry;
312 struct mt76_desc *desc;
313 int i, idx = -1;
314 u32 ctrl, next;
315
316 if (txwi) {
317 q->entry[q->head].txwi = DMA_DUMMY_DATA;
318 q->entry[q->head].skip_buf0 = true;
319 }
320
321 for (i = 0; i < nbufs; i += 2, buf += 2) {
322 u32 buf0 = buf[0].addr, buf1 = 0;
323
324 idx = q->head;
325 next = (q->head + 1) % q->ndesc;
326
327 desc = &q->desc[idx];
328 entry = &q->entry[idx];
329
330 if (buf[0].skip_unmap)
331 entry->skip_buf0 = true;
332 entry->skip_buf1 = i == nbufs - 1;
333
334 entry->dma_addr[0] = buf[0].addr;
335 entry->dma_len[0] = buf[0].len;
336
337 ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
338#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
339 info |= FIELD_PREP(MT_DMA_CTL_SDP0_H, buf[0].addr >> 32);
340#endif
341 if (i < nbufs - 1) {
342 entry->dma_addr[1] = buf[1].addr;
343 entry->dma_len[1] = buf[1].len;
344 buf1 = buf[1].addr;
345 ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
346#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
347 info |= FIELD_PREP(MT_DMA_CTL_SDP1_H,
348 buf[1].addr >> 32);
349#endif
350 if (buf[1].skip_unmap)
351 entry->skip_buf1 = true;
352 }
353
354 if (i == nbufs - 1)
355 ctrl |= MT_DMA_CTL_LAST_SEC0;
356 else if (i == nbufs - 2)
357 ctrl |= MT_DMA_CTL_LAST_SEC1;
358
359 WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
360 WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
361 WRITE_ONCE(desc->info, cpu_to_le32(info));
362 WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
363
364 q->head = next;
365 q->queued++;
366 }
367
368 q->entry[idx].txwi = txwi;
369 q->entry[idx].skb = skb;
370 q->entry[idx].wcid = 0xffff;
371
372 return idx;
373}
374
375static void
376mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
377 struct mt76_queue_entry *prev_e)
378{
379 struct mt76_queue_entry *e = &q->entry[idx];
380
381 if (!e->skip_buf0)
382 dma_unmap_single(dev->dma_dev, e->dma_addr[0], e->dma_len[0],
383 DMA_TO_DEVICE);
384
385 if (!e->skip_buf1)
386 dma_unmap_single(dev->dma_dev, e->dma_addr[1], e->dma_len[1],
387 DMA_TO_DEVICE);
388
389 if (e->txwi == DMA_DUMMY_DATA)
390 e->txwi = NULL;
391
392 *prev_e = *e;
393 memset(e, 0, sizeof(*e));
394}
395
396static void
397mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
398{
399 wmb();
400 if (mt76_queue_is_emi(q))
401 *q->emi_cpu_idx = cpu_to_le16(q->head);
402 else
403 Q_WRITE(q, cpu_idx, q->head);
404}
405
406static void
407mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
408{
409 struct mt76_queue_entry entry;
410 int last;
411
412 if (!q || !q->ndesc)
413 return;
414
415 spin_lock_bh(lock: &q->cleanup_lock);
416 if (flush)
417 last = -1;
418 else
419 last = Q_READ(q, dma_idx);
420
421 while (q->queued > 0 && q->tail != last) {
422 mt76_dma_tx_cleanup_idx(dev, q, idx: q->tail, prev_e: &entry);
423 mt76_npu_txdesc_cleanup(q, index: q->tail);
424 mt76_queue_tx_complete(dev, q, e: &entry);
425
426 if (entry.txwi) {
427 if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
428 mt76_put_txwi(dev, entry.txwi);
429 }
430
431 if (!flush && q->tail == last)
432 last = Q_READ(q, dma_idx);
433 }
434 spin_unlock_bh(lock: &q->cleanup_lock);
435
436 if (flush) {
437 spin_lock_bh(lock: &q->lock);
438 mt76_dma_sync_idx(dev, q);
439 mt76_dma_kick_queue(dev, q);
440 spin_unlock_bh(lock: &q->lock);
441 }
442
443 if (!q->queued)
444 wake_up(&dev->tx_wait);
445}
446
447static void *
448mt76_dma_get_rxdmad_c_buf(struct mt76_dev *dev, struct mt76_queue *q,
449 int idx, int *len, bool *more)
450{
451 struct mt76_queue_entry *e = &q->entry[idx];
452 struct mt76_rro_rxdmad_c *dmad = e->buf;
453 u32 data1 = le32_to_cpu(dmad->data1);
454 u32 data2 = le32_to_cpu(dmad->data2);
455 struct mt76_txwi_cache *t;
456 u16 rx_token_id;
457 u8 ind_reason;
458 void *buf;
459
460 rx_token_id = FIELD_GET(RRO_RXDMAD_DATA2_RX_TOKEN_ID_MASK, data2);
461 t = mt76_rx_token_release(dev, token: rx_token_id);
462 if (!t)
463 return ERR_PTR(error: -EAGAIN);
464
465 q = &dev->q_rx[t->qid];
466 dma_sync_single_for_cpu(dev: dev->dma_dev, addr: t->dma_addr,
467 SKB_WITH_OVERHEAD(q->buf_size),
468 dir: page_pool_get_dma_dir(pool: q->page_pool));
469
470 if (len)
471 *len = FIELD_GET(RRO_RXDMAD_DATA1_SDL0_MASK, data1);
472 if (more)
473 *more = !FIELD_GET(RRO_RXDMAD_DATA1_LS_MASK, data1);
474
475 buf = t->ptr;
476 ind_reason = FIELD_GET(RRO_RXDMAD_DATA2_IND_REASON_MASK, data2);
477 if (ind_reason == MT_DMA_WED_IND_REASON_REPEAT ||
478 ind_reason == MT_DMA_WED_IND_REASON_OLDPKT) {
479 mt76_put_page_pool_buf(buf, allow_direct: false);
480 buf = ERR_PTR(error: -EAGAIN);
481 }
482 t->ptr = NULL;
483 t->dma_addr = 0;
484
485 mt76_put_rxwi(dev, t);
486
487 return buf;
488}
489
490static void *
491mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
492 int *len, u32 *info, bool *more, bool *drop, bool flush)
493{
494 struct mt76_queue_entry *e = &q->entry[idx];
495 struct mt76_desc *desc = &q->desc[idx];
496 u32 ctrl, desc_info, buf1;
497 void *buf = e->buf;
498
499 if (mt76_queue_is_wed_rro_rxdmad_c(q) && !flush)
500 buf = mt76_dma_get_rxdmad_c_buf(dev, q, idx, len, more);
501
502 if (mt76_queue_is_wed_rro(q))
503 goto done;
504
505 ctrl = le32_to_cpu(READ_ONCE(desc->ctrl));
506 if (len) {
507 *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctrl);
508 *more = !(ctrl & MT_DMA_CTL_LAST_SEC0);
509 }
510
511 desc_info = le32_to_cpu(desc->info);
512 if (info)
513 *info = desc_info;
514
515 buf1 = le32_to_cpu(desc->buf1);
516 mt76_dma_should_drop_buf(drop, ctrl, buf1, info: desc_info);
517
518 if (mt76_queue_is_wed_rx(q)) {
519 u32 token = FIELD_GET(MT_DMA_CTL_TOKEN, buf1);
520 struct mt76_txwi_cache *t = mt76_rx_token_release(dev, token);
521
522 if (!t)
523 return NULL;
524
525 dma_sync_single_for_cpu(dev: dev->dma_dev, addr: t->dma_addr,
526 SKB_WITH_OVERHEAD(q->buf_size),
527 dir: page_pool_get_dma_dir(pool: q->page_pool));
528
529 buf = t->ptr;
530 t->dma_addr = 0;
531 t->ptr = NULL;
532
533 mt76_put_rxwi(dev, t);
534 if (drop)
535 *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP);
536 } else {
537 dma_sync_single_for_cpu(dev: dev->dma_dev, addr: e->dma_addr[0],
538 SKB_WITH_OVERHEAD(q->buf_size),
539 dir: page_pool_get_dma_dir(pool: q->page_pool));
540 }
541
542done:
543 e->buf = NULL;
544 return buf;
545}
546
547static void *
548mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
549 int *len, u32 *info, bool *more, bool *drop)
550{
551 int idx = q->tail;
552
553 *more = false;
554 if (!q->queued)
555 return NULL;
556
557 if (mt76_queue_is_wed_rro_data(q) || mt76_queue_is_wed_rro_msdu_pg(q))
558 goto done;
559
560 if (mt76_queue_is_wed_rro_ind(q)) {
561 struct mt76_wed_rro_ind *cmd;
562 u8 magic_cnt;
563
564 if (flush)
565 goto done;
566
567 cmd = q->entry[idx].buf;
568 magic_cnt = FIELD_GET(RRO_IND_DATA1_MAGIC_CNT_MASK,
569 le32_to_cpu(cmd->data1));
570 if (magic_cnt != q->magic_cnt)
571 return NULL;
572
573 if (q->tail == q->ndesc - 1)
574 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_WED_IND_CMD_CNT;
575 } else if (mt76_queue_is_wed_rro_rxdmad_c(q)) {
576 struct mt76_rro_rxdmad_c *dmad;
577 u16 magic_cnt;
578
579 if (flush)
580 goto done;
581
582 dmad = q->entry[idx].buf;
583 magic_cnt = FIELD_GET(RRO_RXDMAD_DATA3_MAGIC_CNT_MASK,
584 le32_to_cpu(dmad->data3));
585 if (magic_cnt != q->magic_cnt)
586 return NULL;
587
588 if (q->tail == q->ndesc - 1)
589 q->magic_cnt = (q->magic_cnt + 1) % MT_DMA_MAGIC_CNT;
590 } else {
591 if (flush)
592 q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
593 else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
594 return NULL;
595 }
596done:
597 q->tail = (q->tail + 1) % q->ndesc;
598 q->queued--;
599
600 return mt76_dma_get_buf(dev, q, idx, len, info, more, drop, flush);
601}
602
603static int
604mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q,
605 struct sk_buff *skb, u32 tx_info)
606{
607 struct mt76_queue_buf buf = {};
608 dma_addr_t addr;
609
610 if (test_bit(MT76_MCU_RESET, &dev->phy.state))
611 goto error;
612
613 if (q->queued + 1 >= q->ndesc - 1)
614 goto error;
615
616 addr = dma_map_single(dev->dma_dev, skb->data, skb->len,
617 DMA_TO_DEVICE);
618 if (unlikely(dma_mapping_error(dev->dma_dev, addr)))
619 goto error;
620
621 buf.addr = addr;
622 buf.len = skb->len;
623
624 spin_lock_bh(lock: &q->lock);
625 mt76_dma_add_buf(dev, q, buf: &buf, nbufs: 1, info: tx_info, skb, NULL);
626 mt76_dma_kick_queue(dev, q);
627 spin_unlock_bh(lock: &q->lock);
628
629 return 0;
630
631error:
632 dev_kfree_skb(skb);
633 return -ENOMEM;
634}
635
636static int
637mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q,
638 enum mt76_txq_id qid, struct sk_buff *skb,
639 struct mt76_wcid *wcid, struct ieee80211_sta *sta)
640{
641 struct ieee80211_tx_status status = {
642 .sta = sta,
643 };
644 struct mt76_tx_info tx_info = {
645 .skb = skb,
646 };
647 struct mt76_dev *dev = phy->dev;
648 struct ieee80211_hw *hw;
649 int len, n = 0, ret = -ENOMEM;
650 struct mt76_txwi_cache *t;
651 struct sk_buff *iter;
652 dma_addr_t addr;
653 u8 *txwi;
654
655 if (test_bit(MT76_RESET, &phy->state))
656 goto free_skb;
657
658 /* TODO: Take into account unlinear skbs */
659 if (mt76_npu_device_active(dev) && skb_linearize(skb))
660 goto free_skb;
661
662 t = mt76_get_txwi(dev);
663 if (!t)
664 goto free_skb;
665
666 txwi = mt76_get_txwi_ptr(dev, t);
667
668 skb->prev = skb->next = NULL;
669 if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
670 mt76_insert_hdr_pad(skb);
671
672 len = skb_headlen(skb);
673 addr = dma_map_single(dev->dma_dev, skb->data, len, DMA_TO_DEVICE);
674 if (unlikely(dma_mapping_error(dev->dma_dev, addr)))
675 goto free;
676
677 tx_info.buf[n].addr = t->dma_addr;
678 tx_info.buf[n++].len = dev->drv->txwi_size;
679 tx_info.buf[n].addr = addr;
680 tx_info.buf[n++].len = len;
681
682 skb_walk_frags(skb, iter) {
683 if (n == ARRAY_SIZE(tx_info.buf))
684 goto unmap;
685
686 addr = dma_map_single(dev->dma_dev, iter->data, iter->len,
687 DMA_TO_DEVICE);
688 if (unlikely(dma_mapping_error(dev->dma_dev, addr)))
689 goto unmap;
690
691 tx_info.buf[n].addr = addr;
692 tx_info.buf[n++].len = iter->len;
693 }
694 tx_info.nbuf = n;
695
696 if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
697 ret = -ENOMEM;
698 goto unmap;
699 }
700
701 dma_sync_single_for_cpu(dev: dev->dma_dev, addr: t->dma_addr, size: dev->drv->txwi_size,
702 dir: DMA_TO_DEVICE);
703 ret = dev->drv->tx_prepare_skb(dev, txwi, qid, wcid, sta, &tx_info);
704 dma_sync_single_for_device(dev: dev->dma_dev, addr: t->dma_addr, size: dev->drv->txwi_size,
705 dir: DMA_TO_DEVICE);
706 if (ret < 0)
707 goto unmap;
708
709 if (mt76_npu_device_active(dev))
710 return mt76_npu_dma_add_buf(phy, q, skb, buf: &tx_info.buf[1], txwi_ptr: txwi);
711
712 return mt76_dma_add_buf(dev, q, buf: tx_info.buf, nbufs: tx_info.nbuf,
713 info: tx_info.info, skb: tx_info.skb, txwi: t);
714
715unmap:
716 for (n--; n > 0; n--)
717 dma_unmap_single(dev->dma_dev, tx_info.buf[n].addr,
718 tx_info.buf[n].len, DMA_TO_DEVICE);
719
720free:
721#ifdef CONFIG_NL80211_TESTMODE
722 /* fix tx_done accounting on queue overflow */
723 if (mt76_is_testmode_skb(dev, skb, hw: &hw)) {
724 struct mt76_phy *phy = hw->priv;
725
726 if (tx_info.skb == phy->test.tx_skb)
727 phy->test.tx_done--;
728 }
729#endif
730
731 mt76_put_txwi(dev, t);
732
733free_skb:
734 status.skb = tx_info.skb;
735 hw = mt76_tx_status_get_hw(dev, skb: tx_info.skb);
736 spin_lock_bh(lock: &dev->rx_lock);
737 ieee80211_tx_status_ext(hw, status: &status);
738 spin_unlock_bh(lock: &dev->rx_lock);
739
740 return ret;
741}
742
743static int
744mt76_dma_rx_fill_buf(struct mt76_dev *dev, struct mt76_queue *q,
745 bool allow_direct)
746{
747 int len = SKB_WITH_OVERHEAD(q->buf_size);
748 int frames = 0;
749
750 if (!q->ndesc)
751 return 0;
752
753 while (q->queued < q->ndesc - 1) {
754 struct mt76_queue_buf qbuf = {};
755 void *buf = NULL;
756 int offset;
757
758 if (mt76_queue_is_wed_rro_ind(q) ||
759 mt76_queue_is_wed_rro_rxdmad_c(q))
760 goto done;
761
762 buf = mt76_get_page_pool_buf(q, offset: &offset, size: q->buf_size);
763 if (!buf)
764 break;
765
766 qbuf.addr = page_pool_get_dma_addr(page: virt_to_head_page(x: buf)) +
767 offset + q->buf_offset;
768done:
769 qbuf.len = len - q->buf_offset;
770 qbuf.skip_unmap = false;
771 if (mt76_dma_add_rx_buf(dev, q, buf: &qbuf, data: buf) < 0) {
772 mt76_put_page_pool_buf(buf, allow_direct);
773 break;
774 }
775 frames++;
776 }
777
778 if (frames || mt76_queue_is_wed_rx(q))
779 mt76_dma_kick_queue(dev, q);
780
781 return frames;
782}
783
784int mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q,
785 bool allow_direct)
786{
787 int frames;
788
789 spin_lock_bh(lock: &q->lock);
790 frames = mt76_dma_rx_fill_buf(dev, q, allow_direct);
791 spin_unlock_bh(lock: &q->lock);
792
793 return frames;
794}
795
796static int
797mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
798 int idx, int n_desc, int bufsize,
799 u32 ring_base)
800{
801 int ret, size;
802
803 spin_lock_init(&q->lock);
804 spin_lock_init(&q->cleanup_lock);
805
806 q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
807 q->ndesc = n_desc;
808 q->buf_size = bufsize;
809 q->hw_idx = idx;
810 q->dev = dev;
811
812 if (mt76_queue_is_wed_rro_ind(q))
813 size = sizeof(struct mt76_wed_rro_desc);
814 else if (mt76_queue_is_npu_tx(q))
815 size = sizeof(struct airoha_npu_tx_dma_desc);
816 else if (mt76_queue_is_npu_rx(q))
817 size = sizeof(struct airoha_npu_rx_dma_desc);
818 else
819 size = sizeof(struct mt76_desc);
820
821 q->desc = dmam_alloc_coherent(dev: dev->dma_dev, size: q->ndesc * size,
822 dma_handle: &q->desc_dma, GFP_KERNEL);
823 if (!q->desc)
824 return -ENOMEM;
825
826 mt76_dma_queue_magic_cnt_init(dev, q);
827 size = q->ndesc * sizeof(*q->entry);
828 q->entry = devm_kzalloc(dev: dev->dev, size, GFP_KERNEL);
829 if (!q->entry)
830 return -ENOMEM;
831
832 ret = mt76_create_page_pool(dev, q);
833 if (ret)
834 return ret;
835
836 mt76_npu_queue_setup(dev, q);
837 ret = mt76_wed_dma_setup(dev, q, reset: false);
838 if (ret)
839 return ret;
840
841 if (mtk_wed_device_active(&dev->mmio.wed)) {
842 if ((mtk_wed_get_rx_capa(dev: &dev->mmio.wed) && mt76_queue_is_wed_rro(q)) ||
843 mt76_queue_is_wed_tx_free(q))
844 return 0;
845 }
846
847 /* HW specific driver is supposed to reset brand-new EMI queues since
848 * it needs to set cpu index pointer.
849 */
850 mt76_dma_queue_reset(dev, q, reset_idx: !mt76_queue_is_emi(q));
851
852 return 0;
853}
854
855static void
856mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
857{
858 void *buf;
859 bool more;
860
861 if (!q->ndesc)
862 return;
863
864 if (mt76_queue_is_npu(q)) {
865 mt76_npu_queue_cleanup(dev, q);
866 return;
867 }
868
869 do {
870 spin_lock_bh(lock: &q->lock);
871 buf = mt76_dma_dequeue(dev, q, flush: true, NULL, NULL, more: &more, NULL);
872 spin_unlock_bh(lock: &q->lock);
873
874 if (!buf)
875 break;
876
877 if (!mt76_queue_is_wed_rro(q))
878 mt76_put_page_pool_buf(buf, allow_direct: false);
879 } while (1);
880
881 spin_lock_bh(lock: &q->lock);
882 if (q->rx_head) {
883 dev_kfree_skb(q->rx_head);
884 q->rx_head = NULL;
885 }
886
887 spin_unlock_bh(lock: &q->lock);
888}
889
890static void
891mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
892{
893 struct mt76_queue *q = &dev->q_rx[qid];
894
895 if (!q->ndesc)
896 return;
897
898 if (!mt76_queue_is_wed_rro_ind(q) &&
899 !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) {
900 int i;
901
902 for (i = 0; i < q->ndesc; i++)
903 q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
904 }
905
906 mt76_dma_rx_cleanup(dev, q);
907
908 /* reset WED rx queues */
909 mt76_wed_dma_setup(dev, q, reset: true);
910
911 if (mt76_queue_is_wed_tx_free(q))
912 return;
913
914 if (mtk_wed_device_active(&dev->mmio.wed) &&
915 mt76_queue_is_wed_rro(q))
916 return;
917
918 mt76_dma_sync_idx(dev, q);
919 if (mt76_queue_is_npu(q))
920 mt76_npu_fill_rx_queue(dev, q);
921 else
922 mt76_dma_rx_fill(dev, q, allow_direct: false);
923}
924
925static void
926mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
927 int len, bool more, u32 info, bool allow_direct)
928{
929 struct sk_buff *skb = q->rx_head;
930 struct skb_shared_info *shinfo = skb_shinfo(skb);
931 int nr_frags = shinfo->nr_frags;
932
933 if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
934 struct page *page = virt_to_head_page(x: data);
935 int offset = data - page_address(page) + q->buf_offset;
936
937 skb_add_rx_frag(skb, i: nr_frags, page, off: offset, size: len, truesize: q->buf_size);
938 } else {
939 mt76_put_page_pool_buf(buf: data, allow_direct);
940 }
941
942 if (more)
943 return;
944
945 q->rx_head = NULL;
946 if (nr_frags < ARRAY_SIZE(shinfo->frags))
947 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info);
948 else
949 dev_kfree_skb(skb);
950}
951
952static int
953mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
954{
955 int len, data_len, done = 0, dma_idx;
956 struct sk_buff *skb;
957 unsigned char *data;
958 bool check_ddone = false;
959 bool allow_direct = !mt76_queue_is_wed_rx(q);
960 bool more;
961
962 if ((q->flags & MT_QFLAG_WED_RRO_EN) ||
963 (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) &&
964 mt76_queue_is_wed_tx_free(q))) {
965 dma_idx = Q_READ(q, dma_idx);
966 check_ddone = true;
967 }
968
969 while (done < budget) {
970 bool drop = false;
971 u32 info;
972
973 if (check_ddone) {
974 if (q->tail == dma_idx)
975 dma_idx = Q_READ(q, dma_idx);
976
977 if (q->tail == dma_idx)
978 break;
979 }
980
981 data = mt76_dma_dequeue(dev, q, flush: false, len: &len, info: &info, more: &more,
982 drop: &drop);
983 if (!data)
984 break;
985
986 if (PTR_ERR(ptr: data) == -EAGAIN) {
987 done++;
988 continue;
989 }
990
991 if (mt76_queue_is_wed_rro_ind(q) && dev->drv->rx_rro_ind_process)
992 dev->drv->rx_rro_ind_process(dev, data);
993
994 if (mt76_queue_is_wed_rro(q) &&
995 !mt76_queue_is_wed_rro_rxdmad_c(q)) {
996 done++;
997 continue;
998 }
999
1000 if (drop)
1001 goto free_frag;
1002
1003 if (q->rx_head)
1004 data_len = q->buf_size;
1005 else
1006 data_len = SKB_WITH_OVERHEAD(q->buf_size);
1007
1008 if (data_len < len + q->buf_offset) {
1009 dev_kfree_skb(q->rx_head);
1010 q->rx_head = NULL;
1011 goto free_frag;
1012 }
1013
1014 if (q->rx_head) {
1015 mt76_add_fragment(dev, q, data, len, more, info,
1016 allow_direct);
1017 continue;
1018 }
1019
1020 if (!more && dev->drv->rx_check &&
1021 !(dev->drv->rx_check(dev, data, len)))
1022 goto free_frag;
1023
1024 skb = napi_build_skb(data, frag_size: q->buf_size);
1025 if (!skb)
1026 goto free_frag;
1027
1028 skb_reserve(skb, len: q->buf_offset);
1029 skb_mark_for_recycle(skb);
1030
1031 *(u32 *)skb->cb = info;
1032
1033 __skb_put(skb, len);
1034 done++;
1035
1036 if (more) {
1037 q->rx_head = skb;
1038 continue;
1039 }
1040
1041 dev->drv->rx_skb(dev, q - dev->q_rx, skb, &info);
1042 continue;
1043
1044free_frag:
1045 mt76_put_page_pool_buf(buf: data, allow_direct);
1046 }
1047
1048 mt76_dma_rx_fill(dev, q, allow_direct: true);
1049 return done;
1050}
1051
1052int mt76_dma_rx_poll(struct napi_struct *napi, int budget)
1053{
1054 struct mt76_dev *dev;
1055 int qid, done = 0, cur;
1056
1057 dev = mt76_priv(dev: napi->dev);
1058 qid = napi - dev->napi;
1059
1060 rcu_read_lock();
1061
1062 do {
1063 cur = mt76_dma_rx_process(dev, q: &dev->q_rx[qid], budget: budget - done);
1064 mt76_rx_poll_complete(dev, q: qid, napi);
1065 done += cur;
1066 } while (cur && done < budget);
1067
1068 rcu_read_unlock();
1069
1070 if (done < budget && napi_complete(n: napi))
1071 dev->drv->rx_poll_complete(dev, qid);
1072
1073 return done;
1074}
1075EXPORT_SYMBOL_GPL(mt76_dma_rx_poll);
1076
1077static void
1078mt76_dma_rx_queue_init(struct mt76_dev *dev, enum mt76_rxq_id qid,
1079 int (*poll)(struct napi_struct *napi, int budget))
1080{
1081 netif_napi_add(dev: dev->napi_dev, napi: &dev->napi[qid], poll);
1082 mt76_dma_rx_fill_buf(dev, q: &dev->q_rx[qid], allow_direct: false);
1083 napi_enable(n: &dev->napi[qid]);
1084}
1085
1086static int
1087mt76_dma_init(struct mt76_dev *dev,
1088 int (*poll)(struct napi_struct *napi, int budget))
1089{
1090 struct mt76_dev **priv;
1091 int i;
1092
1093 dev->napi_dev = alloc_netdev_dummy(sizeof_priv: sizeof(struct mt76_dev *));
1094 if (!dev->napi_dev)
1095 return -ENOMEM;
1096
1097 /* napi_dev private data points to mt76_dev parent, so, mt76_dev
1098 * can be retrieved given napi_dev
1099 */
1100 priv = netdev_priv(dev: dev->napi_dev);
1101 *priv = dev;
1102
1103 dev->tx_napi_dev = alloc_netdev_dummy(sizeof_priv: sizeof(struct mt76_dev *));
1104 if (!dev->tx_napi_dev) {
1105 free_netdev(dev: dev->napi_dev);
1106 return -ENOMEM;
1107 }
1108 priv = netdev_priv(dev: dev->tx_napi_dev);
1109 *priv = dev;
1110
1111 snprintf(buf: dev->napi_dev->name, size: sizeof(dev->napi_dev->name), fmt: "%s",
1112 wiphy_name(wiphy: dev->hw->wiphy));
1113 dev->napi_dev->threaded = 1;
1114 init_completion(x: &dev->mmio.wed_reset);
1115 init_completion(x: &dev->mmio.wed_reset_complete);
1116
1117 mt76_for_each_q_rx(dev, i) {
1118 if (mt76_queue_is_wed_rro(q: &dev->q_rx[i]))
1119 continue;
1120
1121 mt76_dma_rx_queue_init(dev, qid: i, poll);
1122 }
1123
1124 return 0;
1125}
1126
1127static const struct mt76_queue_ops mt76_dma_ops = {
1128 .init = mt76_dma_init,
1129 .alloc = mt76_dma_alloc_queue,
1130 .reset_q = mt76_dma_queue_reset,
1131 .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
1132 .tx_queue_skb = mt76_dma_tx_queue_skb,
1133 .tx_cleanup = mt76_dma_tx_cleanup,
1134 .rx_queue_init = mt76_dma_rx_queue_init,
1135 .rx_cleanup = mt76_dma_rx_cleanup,
1136 .rx_reset = mt76_dma_rx_reset,
1137 .kick = mt76_dma_kick_queue,
1138};
1139
1140void mt76_dma_attach(struct mt76_dev *dev)
1141{
1142 dev->queue_ops = &mt76_dma_ops;
1143}
1144EXPORT_SYMBOL_GPL(mt76_dma_attach);
1145
1146void mt76_dma_cleanup(struct mt76_dev *dev)
1147{
1148 int i;
1149
1150 mt76_worker_disable(w: &dev->tx_worker);
1151 napi_disable(n: &dev->tx_napi);
1152 netif_napi_del(napi: &dev->tx_napi);
1153
1154 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) {
1155 struct mt76_phy *phy = dev->phys[i];
1156 int j;
1157
1158 if (!phy)
1159 continue;
1160
1161 for (j = 0; j < ARRAY_SIZE(phy->q_tx); j++)
1162 mt76_dma_tx_cleanup(dev, q: phy->q_tx[j], flush: true);
1163 }
1164
1165 for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++)
1166 mt76_dma_tx_cleanup(dev, q: dev->q_mcu[i], flush: true);
1167
1168 mt76_for_each_q_rx(dev, i) {
1169 struct mt76_queue *q = &dev->q_rx[i];
1170
1171 if (mtk_wed_device_active(&dev->mmio.wed) &&
1172 mt76_queue_is_wed_rro(q))
1173 continue;
1174
1175 netif_napi_del(napi: &dev->napi[i]);
1176 mt76_dma_rx_cleanup(dev, q);
1177
1178 page_pool_destroy(pool: q->page_pool);
1179 }
1180
1181 if (mtk_wed_device_active(&dev->mmio.wed))
1182 mtk_wed_device_detach(&dev->mmio.wed);
1183
1184 if (mtk_wed_device_active(&dev->mmio.wed_hif2))
1185 mtk_wed_device_detach(&dev->mmio.wed_hif2);
1186
1187 mt76_free_pending_txwi(dev);
1188 mt76_free_pending_rxwi(dev);
1189 free_netdev(dev: dev->napi_dev);
1190 free_netdev(dev: dev->tx_napi_dev);
1191}
1192EXPORT_SYMBOL_GPL(mt76_dma_cleanup);
1193

source code of linux/drivers/net/wireless/mediatek/mt76/dma.c