1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2021, MediaTek Inc. |
4 | * Copyright (c) 2021-2022, Intel Corporation. |
5 | * |
6 | * Authors: |
7 | * Amir Hanania <amir.hanania@intel.com> |
8 | * Haijun Liu <haijun.liu@mediatek.com> |
9 | * Moises Veleta <moises.veleta@intel.com> |
10 | * Ricardo Martinez <ricardo.martinez@linux.intel.com> |
11 | * Sreehari Kancharla <sreehari.kancharla@intel.com> |
12 | * |
13 | * Contributors: |
14 | * Andy Shevchenko <andriy.shevchenko@linux.intel.com> |
15 | * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> |
16 | * Eliot Lee <eliot.lee@intel.com> |
17 | */ |
18 | |
19 | #include <linux/bits.h> |
20 | #include <linux/bitops.h> |
21 | #include <linux/delay.h> |
22 | #include <linux/device.h> |
23 | #include <linux/dmapool.h> |
24 | #include <linux/dma-mapping.h> |
25 | #include <linux/dma-direction.h> |
26 | #include <linux/gfp.h> |
27 | #include <linux/io.h> |
28 | #include <linux/io-64-nonatomic-lo-hi.h> |
29 | #include <linux/iopoll.h> |
30 | #include <linux/irqreturn.h> |
31 | #include <linux/kernel.h> |
32 | #include <linux/kthread.h> |
33 | #include <linux/list.h> |
34 | #include <linux/netdevice.h> |
35 | #include <linux/pci.h> |
36 | #include <linux/pm_runtime.h> |
37 | #include <linux/sched.h> |
38 | #include <linux/skbuff.h> |
39 | #include <linux/slab.h> |
40 | #include <linux/spinlock.h> |
41 | #include <linux/types.h> |
42 | #include <linux/wait.h> |
43 | #include <linux/workqueue.h> |
44 | |
45 | #include "t7xx_cldma.h" |
46 | #include "t7xx_hif_cldma.h" |
47 | #include "t7xx_mhccif.h" |
48 | #include "t7xx_pci.h" |
49 | #include "t7xx_pcie_mac.h" |
50 | #include "t7xx_port_proxy.h" |
51 | #include "t7xx_reg.h" |
52 | #include "t7xx_state_monitor.h" |
53 | |
54 | #define MAX_TX_BUDGET 16 |
55 | #define MAX_RX_BUDGET 16 |
56 | |
57 | #define CHECK_Q_STOP_TIMEOUT_US 1000000 |
58 | #define CHECK_Q_STOP_STEP_US 10000 |
59 | |
60 | static void md_cd_queue_struct_reset(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl, |
61 | enum mtk_txrx tx_rx, unsigned int index) |
62 | { |
63 | queue->dir = tx_rx; |
64 | queue->index = index; |
65 | queue->md_ctrl = md_ctrl; |
66 | queue->tr_ring = NULL; |
67 | queue->tr_done = NULL; |
68 | queue->tx_next = NULL; |
69 | } |
70 | |
71 | static void md_cd_queue_struct_init(struct cldma_queue *queue, struct cldma_ctrl *md_ctrl, |
72 | enum mtk_txrx tx_rx, unsigned int index) |
73 | { |
74 | md_cd_queue_struct_reset(queue, md_ctrl, tx_rx, index); |
75 | init_waitqueue_head(&queue->req_wq); |
76 | spin_lock_init(&queue->ring_lock); |
77 | } |
78 | |
79 | static void t7xx_cldma_gpd_set_data_ptr(struct cldma_gpd *gpd, dma_addr_t data_ptr) |
80 | { |
81 | gpd->data_buff_bd_ptr_h = cpu_to_le32(upper_32_bits(data_ptr)); |
82 | gpd->data_buff_bd_ptr_l = cpu_to_le32(lower_32_bits(data_ptr)); |
83 | } |
84 | |
85 | static void t7xx_cldma_gpd_set_next_ptr(struct cldma_gpd *gpd, dma_addr_t next_ptr) |
86 | { |
87 | gpd->next_gpd_ptr_h = cpu_to_le32(upper_32_bits(next_ptr)); |
88 | gpd->next_gpd_ptr_l = cpu_to_le32(lower_32_bits(next_ptr)); |
89 | } |
90 | |
91 | static int t7xx_cldma_alloc_and_map_skb(struct cldma_ctrl *md_ctrl, struct cldma_request *req, |
92 | size_t size, gfp_t gfp_mask) |
93 | { |
94 | req->skb = __dev_alloc_skb(length: size, gfp_mask); |
95 | if (!req->skb) |
96 | return -ENOMEM; |
97 | |
98 | req->mapped_buff = dma_map_single(md_ctrl->dev, req->skb->data, size, DMA_FROM_DEVICE); |
99 | if (dma_mapping_error(dev: md_ctrl->dev, dma_addr: req->mapped_buff)) { |
100 | dev_kfree_skb_any(skb: req->skb); |
101 | req->skb = NULL; |
102 | req->mapped_buff = 0; |
103 | dev_err(md_ctrl->dev, "DMA mapping failed\n" ); |
104 | return -ENOMEM; |
105 | } |
106 | |
107 | return 0; |
108 | } |
109 | |
110 | static int t7xx_cldma_gpd_rx_from_q(struct cldma_queue *queue, int budget, bool *over_budget) |
111 | { |
112 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
113 | unsigned int hwo_polling_count = 0; |
114 | struct t7xx_cldma_hw *hw_info; |
115 | bool rx_not_done = true; |
116 | unsigned long flags; |
117 | int count = 0; |
118 | |
119 | hw_info = &md_ctrl->hw_info; |
120 | |
121 | do { |
122 | struct cldma_request *req; |
123 | struct cldma_gpd *gpd; |
124 | struct sk_buff *skb; |
125 | int ret; |
126 | |
127 | req = queue->tr_done; |
128 | if (!req) |
129 | return -ENODATA; |
130 | |
131 | gpd = req->gpd; |
132 | if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) { |
133 | dma_addr_t gpd_addr; |
134 | |
135 | if (!pci_device_is_present(to_pci_dev(md_ctrl->dev))) { |
136 | dev_err(md_ctrl->dev, "PCIe Link disconnected\n" ); |
137 | return -ENODEV; |
138 | } |
139 | |
140 | gpd_addr = ioread64_lo_hi(addr: hw_info->ap_pdn_base + |
141 | REG_CLDMA_DL_CURRENT_ADDRL_0 + |
142 | queue->index * sizeof(u64)); |
143 | if (req->gpd_addr == gpd_addr || hwo_polling_count++ >= 100) |
144 | return 0; |
145 | |
146 | udelay(1); |
147 | continue; |
148 | } |
149 | |
150 | hwo_polling_count = 0; |
151 | skb = req->skb; |
152 | |
153 | if (req->mapped_buff) { |
154 | dma_unmap_single(md_ctrl->dev, req->mapped_buff, |
155 | queue->tr_ring->pkt_size, DMA_FROM_DEVICE); |
156 | req->mapped_buff = 0; |
157 | } |
158 | |
159 | skb->len = 0; |
160 | skb_reset_tail_pointer(skb); |
161 | skb_put(skb, le16_to_cpu(gpd->data_buff_len)); |
162 | |
163 | ret = queue->recv_skb(queue, skb); |
164 | /* Break processing, will try again later */ |
165 | if (ret < 0) |
166 | return ret; |
167 | |
168 | req->skb = NULL; |
169 | t7xx_cldma_gpd_set_data_ptr(gpd, data_ptr: 0); |
170 | |
171 | spin_lock_irqsave(&queue->ring_lock, flags); |
172 | queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry); |
173 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
174 | req = queue->rx_refill; |
175 | |
176 | ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, size: queue->tr_ring->pkt_size, GFP_KERNEL); |
177 | if (ret) |
178 | return ret; |
179 | |
180 | gpd = req->gpd; |
181 | t7xx_cldma_gpd_set_data_ptr(gpd, data_ptr: req->mapped_buff); |
182 | gpd->data_buff_len = 0; |
183 | gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO; |
184 | |
185 | spin_lock_irqsave(&queue->ring_lock, flags); |
186 | queue->rx_refill = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry); |
187 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
188 | |
189 | rx_not_done = ++count < budget || !need_resched(); |
190 | } while (rx_not_done); |
191 | |
192 | *over_budget = true; |
193 | return 0; |
194 | } |
195 | |
196 | static int t7xx_cldma_gpd_rx_collect(struct cldma_queue *queue, int budget) |
197 | { |
198 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
199 | struct t7xx_cldma_hw *hw_info; |
200 | unsigned int pending_rx_int; |
201 | bool over_budget = false; |
202 | unsigned long flags; |
203 | int ret; |
204 | |
205 | hw_info = &md_ctrl->hw_info; |
206 | |
207 | do { |
208 | ret = t7xx_cldma_gpd_rx_from_q(queue, budget, over_budget: &over_budget); |
209 | if (ret == -ENODATA) |
210 | return 0; |
211 | else if (ret) |
212 | return ret; |
213 | |
214 | pending_rx_int = 0; |
215 | |
216 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
217 | if (md_ctrl->rxq_active & BIT(queue->index)) { |
218 | if (!t7xx_cldma_hw_queue_status(hw_info, qno: queue->index, tx_rx: MTK_RX)) |
219 | t7xx_cldma_hw_resume_queue(hw_info, qno: queue->index, tx_rx: MTK_RX); |
220 | |
221 | pending_rx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index), |
222 | tx_rx: MTK_RX); |
223 | if (pending_rx_int) { |
224 | t7xx_cldma_hw_rx_done(hw_info, bitmask: pending_rx_int); |
225 | |
226 | if (over_budget) { |
227 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
228 | return -EAGAIN; |
229 | } |
230 | } |
231 | } |
232 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
233 | } while (pending_rx_int); |
234 | |
235 | return 0; |
236 | } |
237 | |
238 | static void t7xx_cldma_rx_done(struct work_struct *work) |
239 | { |
240 | struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work); |
241 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
242 | int value; |
243 | |
244 | value = t7xx_cldma_gpd_rx_collect(queue, budget: queue->budget); |
245 | if (value && md_ctrl->rxq_active & BIT(queue->index)) { |
246 | queue_work(wq: queue->worker, work: &queue->cldma_work); |
247 | return; |
248 | } |
249 | |
250 | t7xx_cldma_clear_ip_busy(hw_info: &md_ctrl->hw_info); |
251 | t7xx_cldma_hw_irq_en_txrx(hw_info: &md_ctrl->hw_info, qno: queue->index, tx_rx: MTK_RX); |
252 | t7xx_cldma_hw_irq_en_eq(hw_info: &md_ctrl->hw_info, qno: queue->index, tx_rx: MTK_RX); |
253 | pm_runtime_mark_last_busy(dev: md_ctrl->dev); |
254 | pm_runtime_put_autosuspend(dev: md_ctrl->dev); |
255 | } |
256 | |
257 | static int t7xx_cldma_gpd_tx_collect(struct cldma_queue *queue) |
258 | { |
259 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
260 | unsigned int dma_len, count = 0; |
261 | struct cldma_request *req; |
262 | struct cldma_gpd *gpd; |
263 | unsigned long flags; |
264 | dma_addr_t dma_free; |
265 | struct sk_buff *skb; |
266 | |
267 | while (!kthread_should_stop()) { |
268 | spin_lock_irqsave(&queue->ring_lock, flags); |
269 | req = queue->tr_done; |
270 | if (!req) { |
271 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
272 | break; |
273 | } |
274 | gpd = req->gpd; |
275 | if ((gpd->flags & GPD_FLAGS_HWO) || !req->skb) { |
276 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
277 | break; |
278 | } |
279 | queue->budget++; |
280 | dma_free = req->mapped_buff; |
281 | dma_len = le16_to_cpu(gpd->data_buff_len); |
282 | skb = req->skb; |
283 | req->skb = NULL; |
284 | queue->tr_done = list_next_entry_circular(req, &queue->tr_ring->gpd_ring, entry); |
285 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
286 | |
287 | count++; |
288 | dma_unmap_single(md_ctrl->dev, dma_free, dma_len, DMA_TO_DEVICE); |
289 | dev_kfree_skb_any(skb); |
290 | } |
291 | |
292 | if (count) |
293 | wake_up_nr(&queue->req_wq, count); |
294 | |
295 | return count; |
296 | } |
297 | |
298 | static void t7xx_cldma_txq_empty_hndl(struct cldma_queue *queue) |
299 | { |
300 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
301 | struct cldma_request *req; |
302 | dma_addr_t ul_curr_addr; |
303 | unsigned long flags; |
304 | bool pending_gpd; |
305 | |
306 | if (!(md_ctrl->txq_active & BIT(queue->index))) |
307 | return; |
308 | |
309 | spin_lock_irqsave(&queue->ring_lock, flags); |
310 | req = list_prev_entry_circular(queue->tx_next, &queue->tr_ring->gpd_ring, entry); |
311 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
312 | |
313 | pending_gpd = (req->gpd->flags & GPD_FLAGS_HWO) && req->skb; |
314 | |
315 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
316 | if (pending_gpd) { |
317 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
318 | |
319 | /* Check current processing TGPD, 64-bit address is in a table by Q index */ |
320 | ul_curr_addr = ioread64_lo_hi(addr: hw_info->ap_pdn_base + REG_CLDMA_UL_CURRENT_ADDRL_0 + |
321 | queue->index * sizeof(u64)); |
322 | if (req->gpd_addr != ul_curr_addr) { |
323 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
324 | dev_err(md_ctrl->dev, "CLDMA%d queue %d is not empty\n" , |
325 | md_ctrl->hif_id, queue->index); |
326 | return; |
327 | } |
328 | |
329 | t7xx_cldma_hw_resume_queue(hw_info, qno: queue->index, tx_rx: MTK_TX); |
330 | } |
331 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
332 | } |
333 | |
334 | static void t7xx_cldma_tx_done(struct work_struct *work) |
335 | { |
336 | struct cldma_queue *queue = container_of(work, struct cldma_queue, cldma_work); |
337 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
338 | struct t7xx_cldma_hw *hw_info; |
339 | unsigned int l2_tx_int; |
340 | unsigned long flags; |
341 | |
342 | hw_info = &md_ctrl->hw_info; |
343 | t7xx_cldma_gpd_tx_collect(queue); |
344 | l2_tx_int = t7xx_cldma_hw_int_status(hw_info, BIT(queue->index) | EQ_STA_BIT(queue->index), |
345 | tx_rx: MTK_TX); |
346 | if (l2_tx_int & EQ_STA_BIT(queue->index)) { |
347 | t7xx_cldma_hw_tx_done(hw_info, EQ_STA_BIT(queue->index)); |
348 | t7xx_cldma_txq_empty_hndl(queue); |
349 | } |
350 | |
351 | if (l2_tx_int & BIT(queue->index)) { |
352 | t7xx_cldma_hw_tx_done(hw_info, BIT(queue->index)); |
353 | queue_work(wq: queue->worker, work: &queue->cldma_work); |
354 | return; |
355 | } |
356 | |
357 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
358 | if (md_ctrl->txq_active & BIT(queue->index)) { |
359 | t7xx_cldma_clear_ip_busy(hw_info); |
360 | t7xx_cldma_hw_irq_en_eq(hw_info, qno: queue->index, tx_rx: MTK_TX); |
361 | t7xx_cldma_hw_irq_en_txrx(hw_info, qno: queue->index, tx_rx: MTK_TX); |
362 | } |
363 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
364 | |
365 | pm_runtime_mark_last_busy(dev: md_ctrl->dev); |
366 | pm_runtime_put_autosuspend(dev: md_ctrl->dev); |
367 | } |
368 | |
369 | static void t7xx_cldma_ring_free(struct cldma_ctrl *md_ctrl, |
370 | struct cldma_ring *ring, enum dma_data_direction tx_rx) |
371 | { |
372 | struct cldma_request *req_cur, *req_next; |
373 | |
374 | list_for_each_entry_safe(req_cur, req_next, &ring->gpd_ring, entry) { |
375 | if (req_cur->mapped_buff && req_cur->skb) { |
376 | dma_unmap_single(md_ctrl->dev, req_cur->mapped_buff, |
377 | ring->pkt_size, tx_rx); |
378 | req_cur->mapped_buff = 0; |
379 | } |
380 | |
381 | dev_kfree_skb_any(skb: req_cur->skb); |
382 | |
383 | if (req_cur->gpd) |
384 | dma_pool_free(pool: md_ctrl->gpd_dmapool, vaddr: req_cur->gpd, addr: req_cur->gpd_addr); |
385 | |
386 | list_del(entry: &req_cur->entry); |
387 | kfree(objp: req_cur); |
388 | } |
389 | } |
390 | |
391 | static struct cldma_request *t7xx_alloc_rx_request(struct cldma_ctrl *md_ctrl, size_t pkt_size) |
392 | { |
393 | struct cldma_request *req; |
394 | int val; |
395 | |
396 | req = kzalloc(size: sizeof(*req), GFP_KERNEL); |
397 | if (!req) |
398 | return NULL; |
399 | |
400 | req->gpd = dma_pool_zalloc(pool: md_ctrl->gpd_dmapool, GFP_KERNEL, handle: &req->gpd_addr); |
401 | if (!req->gpd) |
402 | goto err_free_req; |
403 | |
404 | val = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, size: pkt_size, GFP_KERNEL); |
405 | if (val) |
406 | goto err_free_pool; |
407 | |
408 | return req; |
409 | |
410 | err_free_pool: |
411 | dma_pool_free(pool: md_ctrl->gpd_dmapool, vaddr: req->gpd, addr: req->gpd_addr); |
412 | |
413 | err_free_req: |
414 | kfree(objp: req); |
415 | |
416 | return NULL; |
417 | } |
418 | |
419 | static int t7xx_cldma_rx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring) |
420 | { |
421 | struct cldma_request *req; |
422 | struct cldma_gpd *gpd; |
423 | int i; |
424 | |
425 | INIT_LIST_HEAD(list: &ring->gpd_ring); |
426 | ring->length = MAX_RX_BUDGET; |
427 | |
428 | for (i = 0; i < ring->length; i++) { |
429 | req = t7xx_alloc_rx_request(md_ctrl, pkt_size: ring->pkt_size); |
430 | if (!req) { |
431 | t7xx_cldma_ring_free(md_ctrl, ring, tx_rx: DMA_FROM_DEVICE); |
432 | return -ENOMEM; |
433 | } |
434 | |
435 | gpd = req->gpd; |
436 | t7xx_cldma_gpd_set_data_ptr(gpd, data_ptr: req->mapped_buff); |
437 | gpd->rx_data_allow_len = cpu_to_le16(ring->pkt_size); |
438 | gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO; |
439 | INIT_LIST_HEAD(list: &req->entry); |
440 | list_add_tail(new: &req->entry, head: &ring->gpd_ring); |
441 | } |
442 | |
443 | /* Link previous GPD to next GPD, circular */ |
444 | list_for_each_entry(req, &ring->gpd_ring, entry) { |
445 | t7xx_cldma_gpd_set_next_ptr(gpd, next_ptr: req->gpd_addr); |
446 | gpd = req->gpd; |
447 | } |
448 | |
449 | return 0; |
450 | } |
451 | |
452 | static struct cldma_request *t7xx_alloc_tx_request(struct cldma_ctrl *md_ctrl) |
453 | { |
454 | struct cldma_request *req; |
455 | |
456 | req = kzalloc(size: sizeof(*req), GFP_KERNEL); |
457 | if (!req) |
458 | return NULL; |
459 | |
460 | req->gpd = dma_pool_zalloc(pool: md_ctrl->gpd_dmapool, GFP_KERNEL, handle: &req->gpd_addr); |
461 | if (!req->gpd) { |
462 | kfree(objp: req); |
463 | return NULL; |
464 | } |
465 | |
466 | return req; |
467 | } |
468 | |
469 | static int t7xx_cldma_tx_ring_init(struct cldma_ctrl *md_ctrl, struct cldma_ring *ring) |
470 | { |
471 | struct cldma_request *req; |
472 | struct cldma_gpd *gpd; |
473 | int i; |
474 | |
475 | INIT_LIST_HEAD(list: &ring->gpd_ring); |
476 | ring->length = MAX_TX_BUDGET; |
477 | |
478 | for (i = 0; i < ring->length; i++) { |
479 | req = t7xx_alloc_tx_request(md_ctrl); |
480 | if (!req) { |
481 | t7xx_cldma_ring_free(md_ctrl, ring, tx_rx: DMA_TO_DEVICE); |
482 | return -ENOMEM; |
483 | } |
484 | |
485 | gpd = req->gpd; |
486 | gpd->flags = GPD_FLAGS_IOC; |
487 | INIT_LIST_HEAD(list: &req->entry); |
488 | list_add_tail(new: &req->entry, head: &ring->gpd_ring); |
489 | } |
490 | |
491 | /* Link previous GPD to next GPD, circular */ |
492 | list_for_each_entry(req, &ring->gpd_ring, entry) { |
493 | t7xx_cldma_gpd_set_next_ptr(gpd, next_ptr: req->gpd_addr); |
494 | gpd = req->gpd; |
495 | } |
496 | |
497 | return 0; |
498 | } |
499 | |
500 | /** |
501 | * t7xx_cldma_q_reset() - Reset CLDMA request pointers to their initial values. |
502 | * @queue: Pointer to the queue structure. |
503 | * |
504 | * Called with ring_lock (unless called during initialization phase) |
505 | */ |
506 | static void t7xx_cldma_q_reset(struct cldma_queue *queue) |
507 | { |
508 | struct cldma_request *req; |
509 | |
510 | req = list_first_entry(&queue->tr_ring->gpd_ring, struct cldma_request, entry); |
511 | queue->tr_done = req; |
512 | queue->budget = queue->tr_ring->length; |
513 | |
514 | if (queue->dir == MTK_TX) |
515 | queue->tx_next = req; |
516 | else |
517 | queue->rx_refill = req; |
518 | } |
519 | |
520 | static void t7xx_cldma_rxq_init(struct cldma_queue *queue) |
521 | { |
522 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
523 | |
524 | queue->dir = MTK_RX; |
525 | queue->tr_ring = &md_ctrl->rx_ring[queue->index]; |
526 | t7xx_cldma_q_reset(queue); |
527 | } |
528 | |
529 | static void t7xx_cldma_txq_init(struct cldma_queue *queue) |
530 | { |
531 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
532 | |
533 | queue->dir = MTK_TX; |
534 | queue->tr_ring = &md_ctrl->tx_ring[queue->index]; |
535 | t7xx_cldma_q_reset(queue); |
536 | } |
537 | |
538 | static void t7xx_cldma_enable_irq(struct cldma_ctrl *md_ctrl) |
539 | { |
540 | t7xx_pcie_mac_set_int(t7xx_dev: md_ctrl->t7xx_dev, int_type: md_ctrl->hw_info.phy_interrupt_id); |
541 | } |
542 | |
543 | static void t7xx_cldma_disable_irq(struct cldma_ctrl *md_ctrl) |
544 | { |
545 | t7xx_pcie_mac_clear_int(t7xx_dev: md_ctrl->t7xx_dev, int_type: md_ctrl->hw_info.phy_interrupt_id); |
546 | } |
547 | |
548 | static void t7xx_cldma_irq_work_cb(struct cldma_ctrl *md_ctrl) |
549 | { |
550 | unsigned long l2_tx_int_msk, l2_rx_int_msk, l2_tx_int, l2_rx_int, val; |
551 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
552 | int i; |
553 | |
554 | /* L2 raw interrupt status */ |
555 | l2_tx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TISAR0); |
556 | l2_rx_int = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2RISAR0); |
557 | l2_tx_int_msk = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L2TIMR0); |
558 | l2_rx_int_msk = ioread32(hw_info->ap_ao_base + REG_CLDMA_L2RIMR0); |
559 | l2_tx_int &= ~l2_tx_int_msk; |
560 | l2_rx_int &= ~l2_rx_int_msk; |
561 | |
562 | if (l2_tx_int) { |
563 | if (l2_tx_int & (TQ_ERR_INT_BITMASK | TQ_ACTIVE_START_ERR_INT_BITMASK)) { |
564 | /* Read and clear L3 TX interrupt status */ |
565 | val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0); |
566 | iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR0); |
567 | val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1); |
568 | iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3TISAR1); |
569 | } |
570 | |
571 | t7xx_cldma_hw_tx_done(hw_info, bitmask: l2_tx_int); |
572 | if (l2_tx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) { |
573 | for_each_set_bit(i, &l2_tx_int, L2_INT_BIT_COUNT) { |
574 | if (i < CLDMA_TXQ_NUM) { |
575 | pm_runtime_get(dev: md_ctrl->dev); |
576 | t7xx_cldma_hw_irq_dis_eq(hw_info, qno: i, tx_rx: MTK_TX); |
577 | t7xx_cldma_hw_irq_dis_txrx(hw_info, qno: i, tx_rx: MTK_TX); |
578 | queue_work(wq: md_ctrl->txq[i].worker, |
579 | work: &md_ctrl->txq[i].cldma_work); |
580 | } else { |
581 | t7xx_cldma_txq_empty_hndl(queue: &md_ctrl->txq[i - CLDMA_TXQ_NUM]); |
582 | } |
583 | } |
584 | } |
585 | } |
586 | |
587 | if (l2_rx_int) { |
588 | if (l2_rx_int & (RQ_ERR_INT_BITMASK | RQ_ACTIVE_START_ERR_INT_BITMASK)) { |
589 | /* Read and clear L3 RX interrupt status */ |
590 | val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0); |
591 | iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR0); |
592 | val = ioread32(hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1); |
593 | iowrite32(val, hw_info->ap_pdn_base + REG_CLDMA_L3RISAR1); |
594 | } |
595 | |
596 | t7xx_cldma_hw_rx_done(hw_info, bitmask: l2_rx_int); |
597 | if (l2_rx_int & (TXRX_STATUS_BITMASK | EMPTY_STATUS_BITMASK)) { |
598 | l2_rx_int |= l2_rx_int >> CLDMA_RXQ_NUM; |
599 | for_each_set_bit(i, &l2_rx_int, CLDMA_RXQ_NUM) { |
600 | pm_runtime_get(dev: md_ctrl->dev); |
601 | t7xx_cldma_hw_irq_dis_eq(hw_info, qno: i, tx_rx: MTK_RX); |
602 | t7xx_cldma_hw_irq_dis_txrx(hw_info, qno: i, tx_rx: MTK_RX); |
603 | queue_work(wq: md_ctrl->rxq[i].worker, work: &md_ctrl->rxq[i].cldma_work); |
604 | } |
605 | } |
606 | } |
607 | } |
608 | |
609 | static bool t7xx_cldma_qs_are_active(struct cldma_ctrl *md_ctrl) |
610 | { |
611 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
612 | unsigned int tx_active; |
613 | unsigned int rx_active; |
614 | |
615 | if (!pci_device_is_present(to_pci_dev(md_ctrl->dev))) |
616 | return false; |
617 | |
618 | tx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, tx_rx: MTK_TX); |
619 | rx_active = t7xx_cldma_hw_queue_status(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
620 | |
621 | return tx_active || rx_active; |
622 | } |
623 | |
624 | /** |
625 | * t7xx_cldma_stop() - Stop CLDMA. |
626 | * @md_ctrl: CLDMA context structure. |
627 | * |
628 | * Stop TX and RX queues. Disable L1 and L2 interrupts. |
629 | * Clear status registers. |
630 | * |
631 | * Return: |
632 | * * 0 - Success. |
633 | * * -ERROR - Error code from polling cldma_queues_active. |
634 | */ |
635 | int t7xx_cldma_stop(struct cldma_ctrl *md_ctrl) |
636 | { |
637 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
638 | bool active; |
639 | int i, ret; |
640 | |
641 | md_ctrl->rxq_active = 0; |
642 | t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx: MTK_RX); |
643 | md_ctrl->txq_active = 0; |
644 | t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx: MTK_TX); |
645 | md_ctrl->txq_started = 0; |
646 | t7xx_cldma_disable_irq(md_ctrl); |
647 | t7xx_cldma_hw_stop(hw_info, tx_rx: MTK_RX); |
648 | t7xx_cldma_hw_stop(hw_info, tx_rx: MTK_TX); |
649 | t7xx_cldma_hw_tx_done(hw_info, CLDMA_L2TISAR0_ALL_INT_MASK); |
650 | t7xx_cldma_hw_rx_done(hw_info, CLDMA_L2RISAR0_ALL_INT_MASK); |
651 | |
652 | if (md_ctrl->is_late_init) { |
653 | for (i = 0; i < CLDMA_TXQ_NUM; i++) |
654 | flush_work(work: &md_ctrl->txq[i].cldma_work); |
655 | |
656 | for (i = 0; i < CLDMA_RXQ_NUM; i++) |
657 | flush_work(work: &md_ctrl->rxq[i].cldma_work); |
658 | } |
659 | |
660 | ret = read_poll_timeout(t7xx_cldma_qs_are_active, active, !active, CHECK_Q_STOP_STEP_US, |
661 | CHECK_Q_STOP_TIMEOUT_US, true, md_ctrl); |
662 | if (ret) |
663 | dev_err(md_ctrl->dev, "Could not stop CLDMA%d queues" , md_ctrl->hif_id); |
664 | |
665 | return ret; |
666 | } |
667 | |
668 | static void t7xx_cldma_late_release(struct cldma_ctrl *md_ctrl) |
669 | { |
670 | int i; |
671 | |
672 | if (!md_ctrl->is_late_init) |
673 | return; |
674 | |
675 | for (i = 0; i < CLDMA_TXQ_NUM; i++) |
676 | t7xx_cldma_ring_free(md_ctrl, ring: &md_ctrl->tx_ring[i], tx_rx: DMA_TO_DEVICE); |
677 | |
678 | for (i = 0; i < CLDMA_RXQ_NUM; i++) |
679 | t7xx_cldma_ring_free(md_ctrl, ring: &md_ctrl->rx_ring[i], tx_rx: DMA_FROM_DEVICE); |
680 | |
681 | dma_pool_destroy(pool: md_ctrl->gpd_dmapool); |
682 | md_ctrl->gpd_dmapool = NULL; |
683 | md_ctrl->is_late_init = false; |
684 | } |
685 | |
686 | void t7xx_cldma_reset(struct cldma_ctrl *md_ctrl) |
687 | { |
688 | unsigned long flags; |
689 | int i; |
690 | |
691 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
692 | md_ctrl->txq_active = 0; |
693 | md_ctrl->rxq_active = 0; |
694 | t7xx_cldma_disable_irq(md_ctrl); |
695 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
696 | |
697 | for (i = 0; i < CLDMA_TXQ_NUM; i++) { |
698 | cancel_work_sync(work: &md_ctrl->txq[i].cldma_work); |
699 | |
700 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
701 | md_cd_queue_struct_reset(queue: &md_ctrl->txq[i], md_ctrl, tx_rx: MTK_TX, index: i); |
702 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
703 | } |
704 | |
705 | for (i = 0; i < CLDMA_RXQ_NUM; i++) { |
706 | cancel_work_sync(work: &md_ctrl->rxq[i].cldma_work); |
707 | |
708 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
709 | md_cd_queue_struct_reset(queue: &md_ctrl->rxq[i], md_ctrl, tx_rx: MTK_RX, index: i); |
710 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
711 | } |
712 | |
713 | t7xx_cldma_late_release(md_ctrl); |
714 | } |
715 | |
716 | /** |
717 | * t7xx_cldma_start() - Start CLDMA. |
718 | * @md_ctrl: CLDMA context structure. |
719 | * |
720 | * Set TX/RX start address. |
721 | * Start all RX queues and enable L2 interrupt. |
722 | */ |
723 | void t7xx_cldma_start(struct cldma_ctrl *md_ctrl) |
724 | { |
725 | unsigned long flags; |
726 | |
727 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
728 | if (md_ctrl->is_late_init) { |
729 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
730 | int i; |
731 | |
732 | t7xx_cldma_enable_irq(md_ctrl); |
733 | |
734 | for (i = 0; i < CLDMA_TXQ_NUM; i++) { |
735 | if (md_ctrl->txq[i].tr_done) |
736 | t7xx_cldma_hw_set_start_addr(hw_info, qno: i, |
737 | address: md_ctrl->txq[i].tr_done->gpd_addr, |
738 | tx_rx: MTK_TX); |
739 | } |
740 | |
741 | for (i = 0; i < CLDMA_RXQ_NUM; i++) { |
742 | if (md_ctrl->rxq[i].tr_done) |
743 | t7xx_cldma_hw_set_start_addr(hw_info, qno: i, |
744 | address: md_ctrl->rxq[i].tr_done->gpd_addr, |
745 | tx_rx: MTK_RX); |
746 | } |
747 | |
748 | /* Enable L2 interrupt */ |
749 | t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
750 | t7xx_cldma_hw_start(hw_info); |
751 | md_ctrl->txq_started = 0; |
752 | md_ctrl->txq_active |= TXRX_STATUS_BITMASK; |
753 | md_ctrl->rxq_active |= TXRX_STATUS_BITMASK; |
754 | } |
755 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
756 | } |
757 | |
758 | static void t7xx_cldma_clear_txq(struct cldma_ctrl *md_ctrl, int qnum) |
759 | { |
760 | struct cldma_queue *txq = &md_ctrl->txq[qnum]; |
761 | struct cldma_request *req; |
762 | struct cldma_gpd *gpd; |
763 | unsigned long flags; |
764 | |
765 | spin_lock_irqsave(&txq->ring_lock, flags); |
766 | t7xx_cldma_q_reset(queue: txq); |
767 | list_for_each_entry(req, &txq->tr_ring->gpd_ring, entry) { |
768 | gpd = req->gpd; |
769 | gpd->flags &= ~GPD_FLAGS_HWO; |
770 | t7xx_cldma_gpd_set_data_ptr(gpd, data_ptr: 0); |
771 | gpd->data_buff_len = 0; |
772 | dev_kfree_skb_any(skb: req->skb); |
773 | req->skb = NULL; |
774 | } |
775 | spin_unlock_irqrestore(lock: &txq->ring_lock, flags); |
776 | } |
777 | |
778 | static int t7xx_cldma_clear_rxq(struct cldma_ctrl *md_ctrl, int qnum) |
779 | { |
780 | struct cldma_queue *rxq = &md_ctrl->rxq[qnum]; |
781 | struct cldma_request *req; |
782 | struct cldma_gpd *gpd; |
783 | unsigned long flags; |
784 | int ret = 0; |
785 | |
786 | spin_lock_irqsave(&rxq->ring_lock, flags); |
787 | t7xx_cldma_q_reset(queue: rxq); |
788 | list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) { |
789 | gpd = req->gpd; |
790 | gpd->flags = GPD_FLAGS_IOC | GPD_FLAGS_HWO; |
791 | gpd->data_buff_len = 0; |
792 | |
793 | if (req->skb) { |
794 | req->skb->len = 0; |
795 | skb_reset_tail_pointer(skb: req->skb); |
796 | } |
797 | } |
798 | |
799 | list_for_each_entry(req, &rxq->tr_ring->gpd_ring, entry) { |
800 | if (req->skb) |
801 | continue; |
802 | |
803 | ret = t7xx_cldma_alloc_and_map_skb(md_ctrl, req, size: rxq->tr_ring->pkt_size, GFP_ATOMIC); |
804 | if (ret) |
805 | break; |
806 | |
807 | t7xx_cldma_gpd_set_data_ptr(gpd: req->gpd, data_ptr: req->mapped_buff); |
808 | } |
809 | spin_unlock_irqrestore(lock: &rxq->ring_lock, flags); |
810 | |
811 | return ret; |
812 | } |
813 | |
814 | void t7xx_cldma_clear_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx) |
815 | { |
816 | int i; |
817 | |
818 | if (tx_rx == MTK_TX) { |
819 | for (i = 0; i < CLDMA_TXQ_NUM; i++) |
820 | t7xx_cldma_clear_txq(md_ctrl, qnum: i); |
821 | } else { |
822 | for (i = 0; i < CLDMA_RXQ_NUM; i++) |
823 | t7xx_cldma_clear_rxq(md_ctrl, qnum: i); |
824 | } |
825 | } |
826 | |
827 | void t7xx_cldma_stop_all_qs(struct cldma_ctrl *md_ctrl, enum mtk_txrx tx_rx) |
828 | { |
829 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
830 | unsigned long flags; |
831 | |
832 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
833 | t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx); |
834 | t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx); |
835 | if (tx_rx == MTK_RX) |
836 | md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK; |
837 | else |
838 | md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK; |
839 | t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx); |
840 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
841 | } |
842 | |
843 | static int t7xx_cldma_gpd_handle_tx_request(struct cldma_queue *queue, struct cldma_request *tx_req, |
844 | struct sk_buff *skb) |
845 | { |
846 | struct cldma_ctrl *md_ctrl = queue->md_ctrl; |
847 | struct cldma_gpd *gpd = tx_req->gpd; |
848 | unsigned long flags; |
849 | |
850 | /* Update GPD */ |
851 | tx_req->mapped_buff = dma_map_single(md_ctrl->dev, skb->data, skb->len, DMA_TO_DEVICE); |
852 | |
853 | if (dma_mapping_error(dev: md_ctrl->dev, dma_addr: tx_req->mapped_buff)) { |
854 | dev_err(md_ctrl->dev, "DMA mapping failed\n" ); |
855 | return -ENOMEM; |
856 | } |
857 | |
858 | t7xx_cldma_gpd_set_data_ptr(gpd, data_ptr: tx_req->mapped_buff); |
859 | gpd->data_buff_len = cpu_to_le16(skb->len); |
860 | |
861 | /* This lock must cover TGPD setting, as even without a resume operation, |
862 | * CLDMA can send next HWO=1 if last TGPD just finished. |
863 | */ |
864 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
865 | if (md_ctrl->txq_active & BIT(queue->index)) |
866 | gpd->flags |= GPD_FLAGS_HWO; |
867 | |
868 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
869 | |
870 | tx_req->skb = skb; |
871 | return 0; |
872 | } |
873 | |
874 | /* Called with cldma_lock */ |
875 | static void t7xx_cldma_hw_start_send(struct cldma_ctrl *md_ctrl, int qno, |
876 | struct cldma_request *prev_req) |
877 | { |
878 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
879 | |
880 | /* Check whether the device was powered off (CLDMA start address is not set) */ |
881 | if (!t7xx_cldma_tx_addr_is_set(hw_info, qno)) { |
882 | t7xx_cldma_hw_init(hw_info); |
883 | t7xx_cldma_hw_set_start_addr(hw_info, qno, address: prev_req->gpd_addr, tx_rx: MTK_TX); |
884 | md_ctrl->txq_started &= ~BIT(qno); |
885 | } |
886 | |
887 | if (!t7xx_cldma_hw_queue_status(hw_info, qno, tx_rx: MTK_TX)) { |
888 | if (md_ctrl->txq_started & BIT(qno)) |
889 | t7xx_cldma_hw_resume_queue(hw_info, qno, tx_rx: MTK_TX); |
890 | else |
891 | t7xx_cldma_hw_start_queue(hw_info, qno, tx_rx: MTK_TX); |
892 | |
893 | md_ctrl->txq_started |= BIT(qno); |
894 | } |
895 | } |
896 | |
897 | /** |
898 | * t7xx_cldma_set_recv_skb() - Set the callback to handle RX packets. |
899 | * @queue: CLDMA queue. |
900 | * @recv_skb: Receiving skb callback. |
901 | */ |
902 | void t7xx_cldma_set_recv_skb(struct cldma_queue *queue, |
903 | int (*recv_skb)(struct cldma_queue *queue, struct sk_buff *skb)) |
904 | { |
905 | queue->recv_skb = recv_skb; |
906 | } |
907 | |
908 | /** |
909 | * t7xx_cldma_send_skb() - Send control data to modem. |
910 | * @md_ctrl: CLDMA context structure. |
911 | * @qno: Queue number. |
912 | * @skb: Socket buffer. |
913 | * |
914 | * Return: |
915 | * * 0 - Success. |
916 | * * -ENOMEM - Allocation failure. |
917 | * * -EINVAL - Invalid queue request. |
918 | * * -EIO - Queue is not active. |
919 | * * -ETIMEDOUT - Timeout waiting for the device to wake up. |
920 | */ |
921 | int t7xx_cldma_send_skb(struct cldma_ctrl *md_ctrl, int qno, struct sk_buff *skb) |
922 | { |
923 | struct cldma_request *tx_req; |
924 | struct cldma_queue *queue; |
925 | unsigned long flags; |
926 | int ret; |
927 | |
928 | if (qno >= CLDMA_TXQ_NUM) |
929 | return -EINVAL; |
930 | |
931 | ret = pm_runtime_resume_and_get(dev: md_ctrl->dev); |
932 | if (ret < 0 && ret != -EACCES) |
933 | return ret; |
934 | |
935 | t7xx_pci_disable_sleep(t7xx_dev: md_ctrl->t7xx_dev); |
936 | queue = &md_ctrl->txq[qno]; |
937 | |
938 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
939 | if (!(md_ctrl->txq_active & BIT(qno))) { |
940 | ret = -EIO; |
941 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
942 | goto allow_sleep; |
943 | } |
944 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
945 | |
946 | do { |
947 | spin_lock_irqsave(&queue->ring_lock, flags); |
948 | tx_req = queue->tx_next; |
949 | if (queue->budget > 0 && !tx_req->skb) { |
950 | struct list_head *gpd_ring = &queue->tr_ring->gpd_ring; |
951 | |
952 | queue->budget--; |
953 | t7xx_cldma_gpd_handle_tx_request(queue, tx_req, skb); |
954 | queue->tx_next = list_next_entry_circular(tx_req, gpd_ring, entry); |
955 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
956 | |
957 | if (!t7xx_pci_sleep_disable_complete(t7xx_dev: md_ctrl->t7xx_dev)) { |
958 | ret = -ETIMEDOUT; |
959 | break; |
960 | } |
961 | |
962 | /* Protect the access to the modem for queues operations (resume/start) |
963 | * which access shared locations by all the queues. |
964 | * cldma_lock is independent of ring_lock which is per queue. |
965 | */ |
966 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
967 | t7xx_cldma_hw_start_send(md_ctrl, qno, prev_req: tx_req); |
968 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
969 | |
970 | break; |
971 | } |
972 | spin_unlock_irqrestore(lock: &queue->ring_lock, flags); |
973 | |
974 | if (!t7xx_pci_sleep_disable_complete(t7xx_dev: md_ctrl->t7xx_dev)) { |
975 | ret = -ETIMEDOUT; |
976 | break; |
977 | } |
978 | |
979 | if (!t7xx_cldma_hw_queue_status(hw_info: &md_ctrl->hw_info, qno, tx_rx: MTK_TX)) { |
980 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
981 | t7xx_cldma_hw_resume_queue(hw_info: &md_ctrl->hw_info, qno, tx_rx: MTK_TX); |
982 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
983 | } |
984 | |
985 | ret = wait_event_interruptible_exclusive(queue->req_wq, queue->budget > 0); |
986 | } while (!ret); |
987 | |
988 | allow_sleep: |
989 | t7xx_pci_enable_sleep(t7xx_dev: md_ctrl->t7xx_dev); |
990 | pm_runtime_mark_last_busy(dev: md_ctrl->dev); |
991 | pm_runtime_put_autosuspend(dev: md_ctrl->dev); |
992 | return ret; |
993 | } |
994 | |
995 | static void t7xx_cldma_adjust_config(struct cldma_ctrl *md_ctrl, enum cldma_cfg cfg_id) |
996 | { |
997 | int qno; |
998 | |
999 | for (qno = 0; qno < CLDMA_RXQ_NUM; qno++) { |
1000 | md_ctrl->rx_ring[qno].pkt_size = CLDMA_SHARED_Q_BUFF_SZ; |
1001 | t7xx_cldma_set_recv_skb(queue: &md_ctrl->rxq[qno], recv_skb: t7xx_port_proxy_recv_skb); |
1002 | } |
1003 | |
1004 | md_ctrl->rx_ring[CLDMA_RXQ_NUM - 1].pkt_size = CLDMA_JUMBO_BUFF_SZ; |
1005 | |
1006 | for (qno = 0; qno < CLDMA_TXQ_NUM; qno++) |
1007 | md_ctrl->tx_ring[qno].pkt_size = CLDMA_SHARED_Q_BUFF_SZ; |
1008 | |
1009 | if (cfg_id == CLDMA_DEDICATED_Q_CFG) { |
1010 | md_ctrl->tx_ring[CLDMA_Q_IDX_DUMP].pkt_size = CLDMA_DEDICATED_Q_BUFF_SZ; |
1011 | md_ctrl->rx_ring[CLDMA_Q_IDX_DUMP].pkt_size = CLDMA_DEDICATED_Q_BUFF_SZ; |
1012 | t7xx_cldma_set_recv_skb(queue: &md_ctrl->rxq[CLDMA_Q_IDX_DUMP], |
1013 | recv_skb: t7xx_port_proxy_recv_skb_from_dedicated_queue); |
1014 | } |
1015 | } |
1016 | |
1017 | static int t7xx_cldma_late_init(struct cldma_ctrl *md_ctrl) |
1018 | { |
1019 | char dma_pool_name[32]; |
1020 | int i, j, ret; |
1021 | |
1022 | if (md_ctrl->is_late_init) { |
1023 | dev_err(md_ctrl->dev, "CLDMA late init was already done\n" ); |
1024 | return -EALREADY; |
1025 | } |
1026 | |
1027 | snprintf(buf: dma_pool_name, size: sizeof(dma_pool_name), fmt: "cldma_req_hif%d" , md_ctrl->hif_id); |
1028 | |
1029 | md_ctrl->gpd_dmapool = dma_pool_create(name: dma_pool_name, dev: md_ctrl->dev, |
1030 | size: sizeof(struct cldma_gpd), GPD_DMAPOOL_ALIGN, allocation: 0); |
1031 | if (!md_ctrl->gpd_dmapool) { |
1032 | dev_err(md_ctrl->dev, "DMA pool alloc fail\n" ); |
1033 | return -ENOMEM; |
1034 | } |
1035 | |
1036 | for (i = 0; i < CLDMA_TXQ_NUM; i++) { |
1037 | ret = t7xx_cldma_tx_ring_init(md_ctrl, ring: &md_ctrl->tx_ring[i]); |
1038 | if (ret) { |
1039 | dev_err(md_ctrl->dev, "control TX ring init fail\n" ); |
1040 | goto err_free_tx_ring; |
1041 | } |
1042 | } |
1043 | |
1044 | for (j = 0; j < CLDMA_RXQ_NUM; j++) { |
1045 | ret = t7xx_cldma_rx_ring_init(md_ctrl, ring: &md_ctrl->rx_ring[j]); |
1046 | if (ret) { |
1047 | dev_err(md_ctrl->dev, "Control RX ring init fail\n" ); |
1048 | goto err_free_rx_ring; |
1049 | } |
1050 | } |
1051 | |
1052 | for (i = 0; i < CLDMA_TXQ_NUM; i++) |
1053 | t7xx_cldma_txq_init(queue: &md_ctrl->txq[i]); |
1054 | |
1055 | for (j = 0; j < CLDMA_RXQ_NUM; j++) |
1056 | t7xx_cldma_rxq_init(queue: &md_ctrl->rxq[j]); |
1057 | |
1058 | md_ctrl->is_late_init = true; |
1059 | return 0; |
1060 | |
1061 | err_free_rx_ring: |
1062 | while (j--) |
1063 | t7xx_cldma_ring_free(md_ctrl, ring: &md_ctrl->rx_ring[j], tx_rx: DMA_FROM_DEVICE); |
1064 | |
1065 | err_free_tx_ring: |
1066 | while (i--) |
1067 | t7xx_cldma_ring_free(md_ctrl, ring: &md_ctrl->tx_ring[i], tx_rx: DMA_TO_DEVICE); |
1068 | |
1069 | return ret; |
1070 | } |
1071 | |
1072 | static void __iomem *t7xx_pcie_addr_transfer(void __iomem *addr, u32 addr_trs1, u32 phy_addr) |
1073 | { |
1074 | return addr + phy_addr - addr_trs1; |
1075 | } |
1076 | |
1077 | static void t7xx_hw_info_init(struct cldma_ctrl *md_ctrl) |
1078 | { |
1079 | struct t7xx_addr_base *pbase = &md_ctrl->t7xx_dev->base_addr; |
1080 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
1081 | u32 phy_ao_base, phy_pd_base; |
1082 | |
1083 | hw_info->hw_mode = MODE_BIT_64; |
1084 | |
1085 | if (md_ctrl->hif_id == CLDMA_ID_MD) { |
1086 | phy_ao_base = CLDMA1_AO_BASE; |
1087 | phy_pd_base = CLDMA1_PD_BASE; |
1088 | hw_info->phy_interrupt_id = CLDMA1_INT; |
1089 | } else { |
1090 | phy_ao_base = CLDMA0_AO_BASE; |
1091 | phy_pd_base = CLDMA0_PD_BASE; |
1092 | hw_info->phy_interrupt_id = CLDMA0_INT; |
1093 | } |
1094 | |
1095 | hw_info->ap_ao_base = t7xx_pcie_addr_transfer(addr: pbase->pcie_ext_reg_base, |
1096 | addr_trs1: pbase->pcie_dev_reg_trsl_addr, phy_addr: phy_ao_base); |
1097 | hw_info->ap_pdn_base = t7xx_pcie_addr_transfer(addr: pbase->pcie_ext_reg_base, |
1098 | addr_trs1: pbase->pcie_dev_reg_trsl_addr, phy_addr: phy_pd_base); |
1099 | } |
1100 | |
1101 | static int t7xx_cldma_default_recv_skb(struct cldma_queue *queue, struct sk_buff *skb) |
1102 | { |
1103 | dev_kfree_skb_any(skb); |
1104 | return 0; |
1105 | } |
1106 | |
1107 | int t7xx_cldma_alloc(enum cldma_id hif_id, struct t7xx_pci_dev *t7xx_dev) |
1108 | { |
1109 | struct device *dev = &t7xx_dev->pdev->dev; |
1110 | struct cldma_ctrl *md_ctrl; |
1111 | int qno; |
1112 | |
1113 | md_ctrl = devm_kzalloc(dev, size: sizeof(*md_ctrl), GFP_KERNEL); |
1114 | if (!md_ctrl) |
1115 | return -ENOMEM; |
1116 | |
1117 | md_ctrl->t7xx_dev = t7xx_dev; |
1118 | md_ctrl->dev = dev; |
1119 | md_ctrl->hif_id = hif_id; |
1120 | for (qno = 0; qno < CLDMA_RXQ_NUM; qno++) |
1121 | md_ctrl->rxq[qno].recv_skb = t7xx_cldma_default_recv_skb; |
1122 | |
1123 | t7xx_hw_info_init(md_ctrl); |
1124 | t7xx_dev->md->md_ctrl[hif_id] = md_ctrl; |
1125 | return 0; |
1126 | } |
1127 | |
1128 | static void t7xx_cldma_resume_early(struct t7xx_pci_dev *t7xx_dev, void *entity_param) |
1129 | { |
1130 | struct cldma_ctrl *md_ctrl = entity_param; |
1131 | struct t7xx_cldma_hw *hw_info; |
1132 | unsigned long flags; |
1133 | int qno_t; |
1134 | |
1135 | hw_info = &md_ctrl->hw_info; |
1136 | |
1137 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
1138 | t7xx_cldma_hw_restore(hw_info); |
1139 | for (qno_t = 0; qno_t < CLDMA_TXQ_NUM; qno_t++) { |
1140 | t7xx_cldma_hw_set_start_addr(hw_info, qno: qno_t, address: md_ctrl->txq[qno_t].tx_next->gpd_addr, |
1141 | tx_rx: MTK_TX); |
1142 | t7xx_cldma_hw_set_start_addr(hw_info, qno: qno_t, address: md_ctrl->rxq[qno_t].tr_done->gpd_addr, |
1143 | tx_rx: MTK_RX); |
1144 | } |
1145 | t7xx_cldma_enable_irq(md_ctrl); |
1146 | t7xx_cldma_hw_start_queue(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
1147 | md_ctrl->rxq_active |= TXRX_STATUS_BITMASK; |
1148 | t7xx_cldma_hw_irq_en_eq(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
1149 | t7xx_cldma_hw_irq_en_txrx(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
1150 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
1151 | } |
1152 | |
1153 | static int t7xx_cldma_resume(struct t7xx_pci_dev *t7xx_dev, void *entity_param) |
1154 | { |
1155 | struct cldma_ctrl *md_ctrl = entity_param; |
1156 | unsigned long flags; |
1157 | |
1158 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
1159 | md_ctrl->txq_active |= TXRX_STATUS_BITMASK; |
1160 | t7xx_cldma_hw_irq_en_txrx(hw_info: &md_ctrl->hw_info, CLDMA_ALL_Q, tx_rx: MTK_TX); |
1161 | t7xx_cldma_hw_irq_en_eq(hw_info: &md_ctrl->hw_info, CLDMA_ALL_Q, tx_rx: MTK_TX); |
1162 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
1163 | |
1164 | if (md_ctrl->hif_id == CLDMA_ID_MD) |
1165 | t7xx_mhccif_mask_clr(t7xx_dev, D2H_SW_INT_MASK); |
1166 | |
1167 | return 0; |
1168 | } |
1169 | |
1170 | static void t7xx_cldma_suspend_late(struct t7xx_pci_dev *t7xx_dev, void *entity_param) |
1171 | { |
1172 | struct cldma_ctrl *md_ctrl = entity_param; |
1173 | struct t7xx_cldma_hw *hw_info; |
1174 | unsigned long flags; |
1175 | |
1176 | hw_info = &md_ctrl->hw_info; |
1177 | |
1178 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
1179 | t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
1180 | t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx: MTK_RX); |
1181 | md_ctrl->rxq_active &= ~TXRX_STATUS_BITMASK; |
1182 | t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx: MTK_RX); |
1183 | t7xx_cldma_clear_ip_busy(hw_info); |
1184 | t7xx_cldma_disable_irq(md_ctrl); |
1185 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
1186 | } |
1187 | |
1188 | static int t7xx_cldma_suspend(struct t7xx_pci_dev *t7xx_dev, void *entity_param) |
1189 | { |
1190 | struct cldma_ctrl *md_ctrl = entity_param; |
1191 | struct t7xx_cldma_hw *hw_info; |
1192 | unsigned long flags; |
1193 | |
1194 | if (md_ctrl->hif_id == CLDMA_ID_MD) |
1195 | t7xx_mhccif_mask_set(t7xx_dev, D2H_SW_INT_MASK); |
1196 | |
1197 | hw_info = &md_ctrl->hw_info; |
1198 | |
1199 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
1200 | t7xx_cldma_hw_irq_dis_eq(hw_info, CLDMA_ALL_Q, tx_rx: MTK_TX); |
1201 | t7xx_cldma_hw_irq_dis_txrx(hw_info, CLDMA_ALL_Q, tx_rx: MTK_TX); |
1202 | md_ctrl->txq_active &= ~TXRX_STATUS_BITMASK; |
1203 | t7xx_cldma_hw_stop_all_qs(hw_info, tx_rx: MTK_TX); |
1204 | md_ctrl->txq_started = 0; |
1205 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
1206 | |
1207 | return 0; |
1208 | } |
1209 | |
1210 | static int t7xx_cldma_pm_init(struct cldma_ctrl *md_ctrl) |
1211 | { |
1212 | md_ctrl->pm_entity = kzalloc(size: sizeof(*md_ctrl->pm_entity), GFP_KERNEL); |
1213 | if (!md_ctrl->pm_entity) |
1214 | return -ENOMEM; |
1215 | |
1216 | md_ctrl->pm_entity->entity_param = md_ctrl; |
1217 | |
1218 | if (md_ctrl->hif_id == CLDMA_ID_MD) |
1219 | md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL1; |
1220 | else |
1221 | md_ctrl->pm_entity->id = PM_ENTITY_ID_CTRL2; |
1222 | |
1223 | md_ctrl->pm_entity->suspend = t7xx_cldma_suspend; |
1224 | md_ctrl->pm_entity->suspend_late = t7xx_cldma_suspend_late; |
1225 | md_ctrl->pm_entity->resume = t7xx_cldma_resume; |
1226 | md_ctrl->pm_entity->resume_early = t7xx_cldma_resume_early; |
1227 | |
1228 | return t7xx_pci_pm_entity_register(t7xx_dev: md_ctrl->t7xx_dev, pm_entity: md_ctrl->pm_entity); |
1229 | } |
1230 | |
1231 | static int t7xx_cldma_pm_uninit(struct cldma_ctrl *md_ctrl) |
1232 | { |
1233 | if (!md_ctrl->pm_entity) |
1234 | return -EINVAL; |
1235 | |
1236 | t7xx_pci_pm_entity_unregister(t7xx_dev: md_ctrl->t7xx_dev, pm_entity: md_ctrl->pm_entity); |
1237 | kfree(objp: md_ctrl->pm_entity); |
1238 | md_ctrl->pm_entity = NULL; |
1239 | return 0; |
1240 | } |
1241 | |
1242 | void t7xx_cldma_hif_hw_init(struct cldma_ctrl *md_ctrl) |
1243 | { |
1244 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
1245 | unsigned long flags; |
1246 | |
1247 | spin_lock_irqsave(&md_ctrl->cldma_lock, flags); |
1248 | t7xx_cldma_hw_stop(hw_info, tx_rx: MTK_TX); |
1249 | t7xx_cldma_hw_stop(hw_info, tx_rx: MTK_RX); |
1250 | t7xx_cldma_hw_rx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK); |
1251 | t7xx_cldma_hw_tx_done(hw_info, EMPTY_STATUS_BITMASK | TXRX_STATUS_BITMASK); |
1252 | t7xx_cldma_hw_init(hw_info); |
1253 | spin_unlock_irqrestore(lock: &md_ctrl->cldma_lock, flags); |
1254 | } |
1255 | |
1256 | static irqreturn_t t7xx_cldma_isr_handler(int irq, void *data) |
1257 | { |
1258 | struct cldma_ctrl *md_ctrl = data; |
1259 | u32 interrupt; |
1260 | |
1261 | interrupt = md_ctrl->hw_info.phy_interrupt_id; |
1262 | t7xx_pcie_mac_clear_int(t7xx_dev: md_ctrl->t7xx_dev, int_type: interrupt); |
1263 | t7xx_cldma_irq_work_cb(md_ctrl); |
1264 | t7xx_pcie_mac_clear_int_status(t7xx_dev: md_ctrl->t7xx_dev, int_type: interrupt); |
1265 | t7xx_pcie_mac_set_int(t7xx_dev: md_ctrl->t7xx_dev, int_type: interrupt); |
1266 | return IRQ_HANDLED; |
1267 | } |
1268 | |
1269 | static void t7xx_cldma_destroy_wqs(struct cldma_ctrl *md_ctrl) |
1270 | { |
1271 | int i; |
1272 | |
1273 | for (i = 0; i < CLDMA_TXQ_NUM; i++) { |
1274 | if (md_ctrl->txq[i].worker) { |
1275 | destroy_workqueue(wq: md_ctrl->txq[i].worker); |
1276 | md_ctrl->txq[i].worker = NULL; |
1277 | } |
1278 | } |
1279 | |
1280 | for (i = 0; i < CLDMA_RXQ_NUM; i++) { |
1281 | if (md_ctrl->rxq[i].worker) { |
1282 | destroy_workqueue(wq: md_ctrl->rxq[i].worker); |
1283 | md_ctrl->rxq[i].worker = NULL; |
1284 | } |
1285 | } |
1286 | } |
1287 | |
1288 | /** |
1289 | * t7xx_cldma_init() - Initialize CLDMA. |
1290 | * @md_ctrl: CLDMA context structure. |
1291 | * |
1292 | * Allocate and initialize device power management entity. |
1293 | * Initialize HIF TX/RX queue structure. |
1294 | * Register CLDMA callback ISR with PCIe driver. |
1295 | * |
1296 | * Return: |
1297 | * * 0 - Success. |
1298 | * * -ERROR - Error code from failure sub-initializations. |
1299 | */ |
1300 | int t7xx_cldma_init(struct cldma_ctrl *md_ctrl) |
1301 | { |
1302 | struct t7xx_cldma_hw *hw_info = &md_ctrl->hw_info; |
1303 | int ret, i; |
1304 | |
1305 | md_ctrl->txq_active = 0; |
1306 | md_ctrl->rxq_active = 0; |
1307 | md_ctrl->is_late_init = false; |
1308 | |
1309 | ret = t7xx_cldma_pm_init(md_ctrl); |
1310 | if (ret) |
1311 | return ret; |
1312 | |
1313 | spin_lock_init(&md_ctrl->cldma_lock); |
1314 | |
1315 | for (i = 0; i < CLDMA_TXQ_NUM; i++) { |
1316 | md_cd_queue_struct_init(queue: &md_ctrl->txq[i], md_ctrl, tx_rx: MTK_TX, index: i); |
1317 | md_ctrl->txq[i].worker = |
1318 | alloc_ordered_workqueue("md_hif%d_tx%d_worker" , |
1319 | WQ_MEM_RECLAIM | (i ? 0 : WQ_HIGHPRI), |
1320 | md_ctrl->hif_id, i); |
1321 | if (!md_ctrl->txq[i].worker) |
1322 | goto err_workqueue; |
1323 | |
1324 | INIT_WORK(&md_ctrl->txq[i].cldma_work, t7xx_cldma_tx_done); |
1325 | } |
1326 | |
1327 | for (i = 0; i < CLDMA_RXQ_NUM; i++) { |
1328 | md_cd_queue_struct_init(queue: &md_ctrl->rxq[i], md_ctrl, tx_rx: MTK_RX, index: i); |
1329 | INIT_WORK(&md_ctrl->rxq[i].cldma_work, t7xx_cldma_rx_done); |
1330 | |
1331 | md_ctrl->rxq[i].worker = |
1332 | alloc_ordered_workqueue("md_hif%d_rx%d_worker" , |
1333 | WQ_MEM_RECLAIM, |
1334 | md_ctrl->hif_id, i); |
1335 | if (!md_ctrl->rxq[i].worker) |
1336 | goto err_workqueue; |
1337 | } |
1338 | |
1339 | t7xx_pcie_mac_clear_int(t7xx_dev: md_ctrl->t7xx_dev, int_type: hw_info->phy_interrupt_id); |
1340 | md_ctrl->t7xx_dev->intr_handler[hw_info->phy_interrupt_id] = t7xx_cldma_isr_handler; |
1341 | md_ctrl->t7xx_dev->intr_thread[hw_info->phy_interrupt_id] = NULL; |
1342 | md_ctrl->t7xx_dev->callback_param[hw_info->phy_interrupt_id] = md_ctrl; |
1343 | t7xx_pcie_mac_clear_int_status(t7xx_dev: md_ctrl->t7xx_dev, int_type: hw_info->phy_interrupt_id); |
1344 | return 0; |
1345 | |
1346 | err_workqueue: |
1347 | t7xx_cldma_destroy_wqs(md_ctrl); |
1348 | t7xx_cldma_pm_uninit(md_ctrl); |
1349 | return -ENOMEM; |
1350 | } |
1351 | |
1352 | void t7xx_cldma_switch_cfg(struct cldma_ctrl *md_ctrl, enum cldma_cfg cfg_id) |
1353 | { |
1354 | t7xx_cldma_late_release(md_ctrl); |
1355 | t7xx_cldma_adjust_config(md_ctrl, cfg_id); |
1356 | t7xx_cldma_late_init(md_ctrl); |
1357 | } |
1358 | |
1359 | void t7xx_cldma_exit(struct cldma_ctrl *md_ctrl) |
1360 | { |
1361 | t7xx_cldma_stop(md_ctrl); |
1362 | t7xx_cldma_late_release(md_ctrl); |
1363 | t7xx_cldma_destroy_wqs(md_ctrl); |
1364 | t7xx_cldma_pm_uninit(md_ctrl); |
1365 | } |
1366 | |