1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2021, MediaTek Inc. |
4 | * Copyright (c) 2021-2022, Intel Corporation. |
5 | * |
6 | * Authors: |
7 | * Amir Hanania <amir.hanania@intel.com> |
8 | * Haijun Liu <haijun.liu@mediatek.com> |
9 | * Eliot Lee <eliot.lee@intel.com> |
10 | * Moises Veleta <moises.veleta@intel.com> |
11 | * Ricardo Martinez <ricardo.martinez@linux.intel.com> |
12 | * |
13 | * Contributors: |
14 | * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> |
15 | * Sreehari Kancharla <sreehari.kancharla@intel.com> |
16 | */ |
17 | |
18 | #include <linux/atomic.h> |
19 | #include <linux/bitfield.h> |
20 | #include <linux/delay.h> |
21 | #include <linux/device.h> |
22 | #include <linux/dma-direction.h> |
23 | #include <linux/dma-mapping.h> |
24 | #include <linux/err.h> |
25 | #include <linux/gfp.h> |
26 | #include <linux/kernel.h> |
27 | #include <linux/kthread.h> |
28 | #include <linux/list.h> |
29 | #include <linux/minmax.h> |
30 | #include <linux/netdevice.h> |
31 | #include <linux/pm_runtime.h> |
32 | #include <linux/sched.h> |
33 | #include <linux/spinlock.h> |
34 | #include <linux/skbuff.h> |
35 | #include <linux/types.h> |
36 | #include <linux/wait.h> |
37 | #include <linux/workqueue.h> |
38 | |
39 | #include "t7xx_dpmaif.h" |
40 | #include "t7xx_hif_dpmaif.h" |
41 | #include "t7xx_hif_dpmaif_tx.h" |
42 | #include "t7xx_pci.h" |
43 | |
44 | #define DPMAIF_SKB_TX_BURST_CNT 5 |
45 | #define DPMAIF_DRB_LIST_LEN 6144 |
46 | |
47 | /* DRB dtype */ |
48 | #define DES_DTYP_PD 0 |
49 | #define DES_DTYP_MSG 1 |
50 | |
51 | static unsigned int t7xx_dpmaif_update_drb_rd_idx(struct dpmaif_ctrl *dpmaif_ctrl, |
52 | unsigned int q_num) |
53 | { |
54 | struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; |
55 | unsigned int old_sw_rd_idx, new_hw_rd_idx, drb_cnt; |
56 | unsigned long flags; |
57 | |
58 | if (!txq->que_started) |
59 | return 0; |
60 | |
61 | old_sw_rd_idx = txq->drb_rd_idx; |
62 | new_hw_rd_idx = t7xx_dpmaif_ul_get_rd_idx(hw_info: &dpmaif_ctrl->hw_info, q_num); |
63 | if (new_hw_rd_idx >= DPMAIF_DRB_LIST_LEN) { |
64 | dev_err(dpmaif_ctrl->dev, "Out of range read index: %u\n" , new_hw_rd_idx); |
65 | return 0; |
66 | } |
67 | |
68 | if (old_sw_rd_idx <= new_hw_rd_idx) |
69 | drb_cnt = new_hw_rd_idx - old_sw_rd_idx; |
70 | else |
71 | drb_cnt = txq->drb_size_cnt - old_sw_rd_idx + new_hw_rd_idx; |
72 | |
73 | spin_lock_irqsave(&txq->tx_lock, flags); |
74 | txq->drb_rd_idx = new_hw_rd_idx; |
75 | spin_unlock_irqrestore(lock: &txq->tx_lock, flags); |
76 | |
77 | return drb_cnt; |
78 | } |
79 | |
80 | static unsigned int t7xx_dpmaif_release_tx_buffer(struct dpmaif_ctrl *dpmaif_ctrl, |
81 | unsigned int q_num, unsigned int release_cnt) |
82 | { |
83 | struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; |
84 | struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; |
85 | struct dpmaif_drb_skb *cur_drb_skb, *drb_skb_base; |
86 | struct dpmaif_drb *cur_drb, *drb_base; |
87 | unsigned int drb_cnt, i, cur_idx; |
88 | unsigned long flags; |
89 | |
90 | drb_skb_base = txq->drb_skb_base; |
91 | drb_base = txq->drb_base; |
92 | |
93 | spin_lock_irqsave(&txq->tx_lock, flags); |
94 | drb_cnt = txq->drb_size_cnt; |
95 | cur_idx = txq->drb_release_rd_idx; |
96 | spin_unlock_irqrestore(lock: &txq->tx_lock, flags); |
97 | |
98 | for (i = 0; i < release_cnt; i++) { |
99 | cur_drb = drb_base + cur_idx; |
100 | if (FIELD_GET(DRB_HDR_DTYP, le32_to_cpu(cur_drb->header)) == DES_DTYP_PD) { |
101 | cur_drb_skb = drb_skb_base + cur_idx; |
102 | if (!cur_drb_skb->is_msg) |
103 | dma_unmap_single(dpmaif_ctrl->dev, cur_drb_skb->bus_addr, |
104 | cur_drb_skb->data_len, DMA_TO_DEVICE); |
105 | |
106 | if (!FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) { |
107 | if (!cur_drb_skb->skb) { |
108 | dev_err(dpmaif_ctrl->dev, |
109 | "txq%u: DRB check fail, invalid skb\n" , q_num); |
110 | continue; |
111 | } |
112 | |
113 | dev_kfree_skb_any(skb: cur_drb_skb->skb); |
114 | } |
115 | |
116 | cur_drb_skb->skb = NULL; |
117 | } |
118 | |
119 | spin_lock_irqsave(&txq->tx_lock, flags); |
120 | cur_idx = t7xx_ring_buf_get_next_wr_idx(buf_len: drb_cnt, buf_idx: cur_idx); |
121 | txq->drb_release_rd_idx = cur_idx; |
122 | spin_unlock_irqrestore(lock: &txq->tx_lock, flags); |
123 | |
124 | if (atomic_inc_return(v: &txq->tx_budget) > txq->drb_size_cnt / 8) |
125 | cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_IRQ, txq->index); |
126 | } |
127 | |
128 | if (FIELD_GET(DRB_HDR_CONT, le32_to_cpu(cur_drb->header))) |
129 | dev_err(dpmaif_ctrl->dev, "txq%u: DRB not marked as the last one\n" , q_num); |
130 | |
131 | return i; |
132 | } |
133 | |
134 | static int t7xx_dpmaif_tx_release(struct dpmaif_ctrl *dpmaif_ctrl, |
135 | unsigned int q_num, unsigned int budget) |
136 | { |
137 | struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[q_num]; |
138 | unsigned int rel_cnt, real_rel_cnt; |
139 | |
140 | /* Update read index from HW */ |
141 | t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl, q_num); |
142 | |
143 | rel_cnt = t7xx_ring_buf_rd_wr_count(total_cnt: txq->drb_size_cnt, rd_idx: txq->drb_release_rd_idx, |
144 | wr_idx: txq->drb_rd_idx, DPMAIF_READ); |
145 | |
146 | real_rel_cnt = min_not_zero(budget, rel_cnt); |
147 | if (real_rel_cnt) |
148 | real_rel_cnt = t7xx_dpmaif_release_tx_buffer(dpmaif_ctrl, q_num, release_cnt: real_rel_cnt); |
149 | |
150 | return real_rel_cnt < rel_cnt ? -EAGAIN : 0; |
151 | } |
152 | |
153 | static bool t7xx_dpmaif_drb_ring_not_empty(struct dpmaif_tx_queue *txq) |
154 | { |
155 | return !!t7xx_dpmaif_update_drb_rd_idx(dpmaif_ctrl: txq->dpmaif_ctrl, q_num: txq->index); |
156 | } |
157 | |
158 | static void t7xx_dpmaif_tx_done(struct work_struct *work) |
159 | { |
160 | struct dpmaif_tx_queue *txq = container_of(work, struct dpmaif_tx_queue, dpmaif_tx_work); |
161 | struct dpmaif_ctrl *dpmaif_ctrl = txq->dpmaif_ctrl; |
162 | struct dpmaif_hw_info *hw_info; |
163 | int ret; |
164 | |
165 | ret = pm_runtime_resume_and_get(dev: dpmaif_ctrl->dev); |
166 | if (ret < 0 && ret != -EACCES) |
167 | return; |
168 | |
169 | /* The device may be in low power state. Disable sleep if needed */ |
170 | t7xx_pci_disable_sleep(t7xx_dev: dpmaif_ctrl->t7xx_dev); |
171 | if (t7xx_pci_sleep_disable_complete(t7xx_dev: dpmaif_ctrl->t7xx_dev)) { |
172 | hw_info = &dpmaif_ctrl->hw_info; |
173 | ret = t7xx_dpmaif_tx_release(dpmaif_ctrl, q_num: txq->index, budget: txq->drb_size_cnt); |
174 | if (ret == -EAGAIN || |
175 | (t7xx_dpmaif_ul_clr_done(hw_info, qno: txq->index) && |
176 | t7xx_dpmaif_drb_ring_not_empty(txq))) { |
177 | queue_work(wq: dpmaif_ctrl->txq[txq->index].worker, |
178 | work: &dpmaif_ctrl->txq[txq->index].dpmaif_tx_work); |
179 | /* Give the device time to enter the low power state */ |
180 | t7xx_dpmaif_clr_ip_busy_sts(hw_info); |
181 | } else { |
182 | t7xx_dpmaif_clr_ip_busy_sts(hw_info); |
183 | t7xx_dpmaif_unmask_ulq_intr(hw_info, q_num: txq->index); |
184 | } |
185 | } |
186 | |
187 | t7xx_pci_enable_sleep(t7xx_dev: dpmaif_ctrl->t7xx_dev); |
188 | pm_runtime_mark_last_busy(dev: dpmaif_ctrl->dev); |
189 | pm_runtime_put_autosuspend(dev: dpmaif_ctrl->dev); |
190 | } |
191 | |
192 | static void t7xx_setup_msg_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, |
193 | unsigned int cur_idx, unsigned int pkt_len, unsigned int count_l, |
194 | unsigned int channel_id) |
195 | { |
196 | struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; |
197 | struct dpmaif_drb *drb = drb_base + cur_idx; |
198 | |
199 | drb->header = cpu_to_le32(FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_MSG) | |
200 | FIELD_PREP(DRB_HDR_CONT, 1) | |
201 | FIELD_PREP(DRB_HDR_DATA_LEN, pkt_len)); |
202 | |
203 | drb->msg.msg_hdr = cpu_to_le32(FIELD_PREP(DRB_MSG_COUNT_L, count_l) | |
204 | FIELD_PREP(DRB_MSG_CHANNEL_ID, channel_id) | |
205 | FIELD_PREP(DRB_MSG_L4_CHK, 1)); |
206 | } |
207 | |
208 | static void t7xx_setup_payload_drb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, |
209 | unsigned int cur_idx, dma_addr_t data_addr, |
210 | unsigned int pkt_size, bool last_one) |
211 | { |
212 | struct dpmaif_drb *drb_base = dpmaif_ctrl->txq[q_num].drb_base; |
213 | struct dpmaif_drb *drb = drb_base + cur_idx; |
214 | u32 ; |
215 | |
216 | header = FIELD_PREP(DRB_HDR_DTYP, DES_DTYP_PD) | FIELD_PREP(DRB_HDR_DATA_LEN, pkt_size); |
217 | if (!last_one) |
218 | header |= FIELD_PREP(DRB_HDR_CONT, 1); |
219 | |
220 | drb->header = cpu_to_le32(header); |
221 | drb->pd.data_addr_l = cpu_to_le32(lower_32_bits(data_addr)); |
222 | drb->pd.data_addr_h = cpu_to_le32(upper_32_bits(data_addr)); |
223 | } |
224 | |
225 | static void t7xx_record_drb_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int q_num, |
226 | unsigned int cur_idx, struct sk_buff *skb, bool is_msg, |
227 | bool is_frag, bool is_last_one, dma_addr_t bus_addr, |
228 | unsigned int data_len) |
229 | { |
230 | struct dpmaif_drb_skb *drb_skb_base = dpmaif_ctrl->txq[q_num].drb_skb_base; |
231 | struct dpmaif_drb_skb *drb_skb = drb_skb_base + cur_idx; |
232 | |
233 | drb_skb->skb = skb; |
234 | drb_skb->bus_addr = bus_addr; |
235 | drb_skb->data_len = data_len; |
236 | drb_skb->index = cur_idx; |
237 | drb_skb->is_msg = is_msg; |
238 | drb_skb->is_frag = is_frag; |
239 | drb_skb->is_last = is_last_one; |
240 | } |
241 | |
242 | static int t7xx_dpmaif_add_skb_to_ring(struct dpmaif_ctrl *dpmaif_ctrl, struct sk_buff *skb) |
243 | { |
244 | struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; |
245 | unsigned int wr_cnt, send_cnt, payload_cnt; |
246 | unsigned int cur_idx, drb_wr_idx_backup; |
247 | struct skb_shared_info *shinfo; |
248 | struct dpmaif_tx_queue *txq; |
249 | struct t7xx_skb_cb *skb_cb; |
250 | unsigned long flags; |
251 | |
252 | skb_cb = T7XX_SKB_CB(skb); |
253 | txq = &dpmaif_ctrl->txq[skb_cb->txq_number]; |
254 | if (!txq->que_started || dpmaif_ctrl->state != DPMAIF_STATE_PWRON) |
255 | return -ENODEV; |
256 | |
257 | atomic_set(v: &txq->tx_processing, i: 1); |
258 | /* Ensure tx_processing is changed to 1 before actually begin TX flow */ |
259 | smp_mb(); |
260 | |
261 | shinfo = skb_shinfo(skb); |
262 | if (shinfo->frag_list) |
263 | dev_warn_ratelimited(dpmaif_ctrl->dev, "frag_list not supported\n" ); |
264 | |
265 | payload_cnt = shinfo->nr_frags + 1; |
266 | /* nr_frags: frag cnt, 1: skb->data, 1: msg DRB */ |
267 | send_cnt = payload_cnt + 1; |
268 | |
269 | spin_lock_irqsave(&txq->tx_lock, flags); |
270 | cur_idx = txq->drb_wr_idx; |
271 | drb_wr_idx_backup = cur_idx; |
272 | txq->drb_wr_idx += send_cnt; |
273 | if (txq->drb_wr_idx >= txq->drb_size_cnt) |
274 | txq->drb_wr_idx -= txq->drb_size_cnt; |
275 | t7xx_setup_msg_drb(dpmaif_ctrl, q_num: txq->index, cur_idx, pkt_len: skb->len, count_l: 0, channel_id: skb_cb->netif_idx); |
276 | t7xx_record_drb_skb(dpmaif_ctrl, q_num: txq->index, cur_idx, skb, is_msg: true, is_frag: 0, is_last_one: 0, bus_addr: 0, data_len: 0); |
277 | spin_unlock_irqrestore(lock: &txq->tx_lock, flags); |
278 | |
279 | for (wr_cnt = 0; wr_cnt < payload_cnt; wr_cnt++) { |
280 | bool is_frag, is_last_one = wr_cnt == payload_cnt - 1; |
281 | unsigned int data_len; |
282 | dma_addr_t bus_addr; |
283 | void *data_addr; |
284 | |
285 | if (!wr_cnt) { |
286 | data_len = skb_headlen(skb); |
287 | data_addr = skb->data; |
288 | is_frag = false; |
289 | } else { |
290 | skb_frag_t *frag = shinfo->frags + wr_cnt - 1; |
291 | |
292 | data_len = skb_frag_size(frag); |
293 | data_addr = skb_frag_address(frag); |
294 | is_frag = true; |
295 | } |
296 | |
297 | bus_addr = dma_map_single(dpmaif_ctrl->dev, data_addr, data_len, DMA_TO_DEVICE); |
298 | if (dma_mapping_error(dev: dpmaif_ctrl->dev, dma_addr: bus_addr)) |
299 | goto unmap_buffers; |
300 | |
301 | cur_idx = t7xx_ring_buf_get_next_wr_idx(buf_len: txq->drb_size_cnt, buf_idx: cur_idx); |
302 | |
303 | spin_lock_irqsave(&txq->tx_lock, flags); |
304 | t7xx_setup_payload_drb(dpmaif_ctrl, q_num: txq->index, cur_idx, data_addr: bus_addr, pkt_size: data_len, |
305 | last_one: is_last_one); |
306 | t7xx_record_drb_skb(dpmaif_ctrl, q_num: txq->index, cur_idx, skb, is_msg: false, is_frag, |
307 | is_last_one, bus_addr, data_len); |
308 | spin_unlock_irqrestore(lock: &txq->tx_lock, flags); |
309 | } |
310 | |
311 | if (atomic_sub_return(i: send_cnt, v: &txq->tx_budget) <= (MAX_SKB_FRAGS + 2)) |
312 | cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq->index); |
313 | |
314 | atomic_set(v: &txq->tx_processing, i: 0); |
315 | |
316 | return 0; |
317 | |
318 | unmap_buffers: |
319 | while (wr_cnt--) { |
320 | struct dpmaif_drb_skb *drb_skb = txq->drb_skb_base; |
321 | |
322 | cur_idx = cur_idx ? cur_idx - 1 : txq->drb_size_cnt - 1; |
323 | drb_skb += cur_idx; |
324 | dma_unmap_single(dpmaif_ctrl->dev, drb_skb->bus_addr, |
325 | drb_skb->data_len, DMA_TO_DEVICE); |
326 | } |
327 | |
328 | txq->drb_wr_idx = drb_wr_idx_backup; |
329 | atomic_set(v: &txq->tx_processing, i: 0); |
330 | |
331 | return -ENOMEM; |
332 | } |
333 | |
334 | static bool t7xx_tx_lists_are_all_empty(const struct dpmaif_ctrl *dpmaif_ctrl) |
335 | { |
336 | int i; |
337 | |
338 | for (i = 0; i < DPMAIF_TXQ_NUM; i++) { |
339 | if (!skb_queue_empty(list: &dpmaif_ctrl->txq[i].tx_skb_head)) |
340 | return false; |
341 | } |
342 | |
343 | return true; |
344 | } |
345 | |
346 | /* Currently, only the default TX queue is used */ |
347 | static struct dpmaif_tx_queue *t7xx_select_tx_queue(struct dpmaif_ctrl *dpmaif_ctrl) |
348 | { |
349 | struct dpmaif_tx_queue *txq; |
350 | |
351 | txq = &dpmaif_ctrl->txq[DPMAIF_TX_DEFAULT_QUEUE]; |
352 | if (!txq->que_started) |
353 | return NULL; |
354 | |
355 | return txq; |
356 | } |
357 | |
358 | static unsigned int t7xx_txq_drb_wr_available(struct dpmaif_tx_queue *txq) |
359 | { |
360 | return t7xx_ring_buf_rd_wr_count(total_cnt: txq->drb_size_cnt, rd_idx: txq->drb_release_rd_idx, |
361 | wr_idx: txq->drb_wr_idx, DPMAIF_WRITE); |
362 | } |
363 | |
364 | static unsigned int t7xx_skb_drb_cnt(struct sk_buff *skb) |
365 | { |
366 | /* Normal DRB (frags data + skb linear data) + msg DRB */ |
367 | return skb_shinfo(skb)->nr_frags + 2; |
368 | } |
369 | |
370 | static int t7xx_txq_burst_send_skb(struct dpmaif_tx_queue *txq) |
371 | { |
372 | unsigned int drb_remain_cnt, i; |
373 | unsigned int send_drb_cnt; |
374 | int drb_cnt = 0; |
375 | int ret = 0; |
376 | |
377 | drb_remain_cnt = t7xx_txq_drb_wr_available(txq); |
378 | |
379 | for (i = 0; i < DPMAIF_SKB_TX_BURST_CNT; i++) { |
380 | struct sk_buff *skb; |
381 | |
382 | skb = skb_peek(list_: &txq->tx_skb_head); |
383 | if (!skb) |
384 | break; |
385 | |
386 | send_drb_cnt = t7xx_skb_drb_cnt(skb); |
387 | if (drb_remain_cnt < send_drb_cnt) { |
388 | drb_remain_cnt = t7xx_txq_drb_wr_available(txq); |
389 | continue; |
390 | } |
391 | |
392 | drb_remain_cnt -= send_drb_cnt; |
393 | |
394 | ret = t7xx_dpmaif_add_skb_to_ring(dpmaif_ctrl: txq->dpmaif_ctrl, skb); |
395 | if (ret < 0) { |
396 | dev_err(txq->dpmaif_ctrl->dev, |
397 | "Failed to add skb to device's ring: %d\n" , ret); |
398 | break; |
399 | } |
400 | |
401 | drb_cnt += send_drb_cnt; |
402 | skb_unlink(skb, list: &txq->tx_skb_head); |
403 | } |
404 | |
405 | if (drb_cnt > 0) |
406 | return drb_cnt; |
407 | |
408 | return ret; |
409 | } |
410 | |
411 | static void t7xx_do_tx_hw_push(struct dpmaif_ctrl *dpmaif_ctrl) |
412 | { |
413 | bool wait_disable_sleep = true; |
414 | |
415 | do { |
416 | struct dpmaif_tx_queue *txq; |
417 | int drb_send_cnt; |
418 | |
419 | txq = t7xx_select_tx_queue(dpmaif_ctrl); |
420 | if (!txq) |
421 | return; |
422 | |
423 | drb_send_cnt = t7xx_txq_burst_send_skb(txq); |
424 | if (drb_send_cnt <= 0) { |
425 | usleep_range(min: 10, max: 20); |
426 | cond_resched(); |
427 | continue; |
428 | } |
429 | |
430 | /* Wait for the PCIe resource to unlock */ |
431 | if (wait_disable_sleep) { |
432 | if (!t7xx_pci_sleep_disable_complete(t7xx_dev: dpmaif_ctrl->t7xx_dev)) |
433 | return; |
434 | |
435 | wait_disable_sleep = false; |
436 | } |
437 | |
438 | t7xx_dpmaif_ul_update_hw_drb_cnt(hw_info: &dpmaif_ctrl->hw_info, q_num: txq->index, |
439 | drb_entry_cnt: drb_send_cnt * DPMAIF_UL_DRB_SIZE_WORD); |
440 | |
441 | cond_resched(); |
442 | } while (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && !kthread_should_stop() && |
443 | (dpmaif_ctrl->state == DPMAIF_STATE_PWRON)); |
444 | } |
445 | |
446 | static int t7xx_dpmaif_tx_hw_push_thread(void *arg) |
447 | { |
448 | struct dpmaif_ctrl *dpmaif_ctrl = arg; |
449 | int ret; |
450 | |
451 | while (!kthread_should_stop()) { |
452 | if (t7xx_tx_lists_are_all_empty(dpmaif_ctrl) || |
453 | dpmaif_ctrl->state != DPMAIF_STATE_PWRON) { |
454 | if (wait_event_interruptible(dpmaif_ctrl->tx_wq, |
455 | (!t7xx_tx_lists_are_all_empty(dpmaif_ctrl) && |
456 | dpmaif_ctrl->state == DPMAIF_STATE_PWRON) || |
457 | kthread_should_stop())) |
458 | continue; |
459 | |
460 | if (kthread_should_stop()) |
461 | break; |
462 | } |
463 | |
464 | ret = pm_runtime_resume_and_get(dev: dpmaif_ctrl->dev); |
465 | if (ret < 0 && ret != -EACCES) |
466 | return ret; |
467 | |
468 | t7xx_pci_disable_sleep(t7xx_dev: dpmaif_ctrl->t7xx_dev); |
469 | t7xx_do_tx_hw_push(dpmaif_ctrl); |
470 | t7xx_pci_enable_sleep(t7xx_dev: dpmaif_ctrl->t7xx_dev); |
471 | pm_runtime_mark_last_busy(dev: dpmaif_ctrl->dev); |
472 | pm_runtime_put_autosuspend(dev: dpmaif_ctrl->dev); |
473 | } |
474 | |
475 | return 0; |
476 | } |
477 | |
478 | int t7xx_dpmaif_tx_thread_init(struct dpmaif_ctrl *dpmaif_ctrl) |
479 | { |
480 | init_waitqueue_head(&dpmaif_ctrl->tx_wq); |
481 | dpmaif_ctrl->tx_thread = kthread_run(t7xx_dpmaif_tx_hw_push_thread, |
482 | dpmaif_ctrl, "dpmaif_tx_hw_push" ); |
483 | return PTR_ERR_OR_ZERO(ptr: dpmaif_ctrl->tx_thread); |
484 | } |
485 | |
486 | void t7xx_dpmaif_tx_thread_rel(struct dpmaif_ctrl *dpmaif_ctrl) |
487 | { |
488 | if (dpmaif_ctrl->tx_thread) |
489 | kthread_stop(k: dpmaif_ctrl->tx_thread); |
490 | } |
491 | |
492 | /** |
493 | * t7xx_dpmaif_tx_send_skb() - Add skb to the transmit queue. |
494 | * @dpmaif_ctrl: Pointer to struct dpmaif_ctrl. |
495 | * @txq_number: Queue number to xmit on. |
496 | * @skb: Pointer to the skb to transmit. |
497 | * |
498 | * Add the skb to the queue of the skbs to be transmit. |
499 | * Wake up the thread that push the skbs from the queue to the HW. |
500 | * |
501 | * Return: |
502 | * * 0 - Success. |
503 | * * -EBUSY - Tx budget exhausted. |
504 | * In normal circumstances t7xx_dpmaif_add_skb_to_ring() must report the txq full |
505 | * state to prevent this error condition. |
506 | */ |
507 | int t7xx_dpmaif_tx_send_skb(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int txq_number, |
508 | struct sk_buff *skb) |
509 | { |
510 | struct dpmaif_tx_queue *txq = &dpmaif_ctrl->txq[txq_number]; |
511 | struct dpmaif_callbacks *cb = dpmaif_ctrl->callbacks; |
512 | struct t7xx_skb_cb *skb_cb; |
513 | |
514 | if (atomic_read(v: &txq->tx_budget) <= t7xx_skb_drb_cnt(skb)) { |
515 | cb->state_notify(dpmaif_ctrl->t7xx_dev, DMPAIF_TXQ_STATE_FULL, txq_number); |
516 | return -EBUSY; |
517 | } |
518 | |
519 | skb_cb = T7XX_SKB_CB(skb); |
520 | skb_cb->txq_number = txq_number; |
521 | skb_queue_tail(list: &txq->tx_skb_head, newsk: skb); |
522 | wake_up(&dpmaif_ctrl->tx_wq); |
523 | |
524 | return 0; |
525 | } |
526 | |
527 | void t7xx_dpmaif_irq_tx_done(struct dpmaif_ctrl *dpmaif_ctrl, unsigned int que_mask) |
528 | { |
529 | int i; |
530 | |
531 | for (i = 0; i < DPMAIF_TXQ_NUM; i++) { |
532 | if (que_mask & BIT(i)) |
533 | queue_work(wq: dpmaif_ctrl->txq[i].worker, work: &dpmaif_ctrl->txq[i].dpmaif_tx_work); |
534 | } |
535 | } |
536 | |
537 | static int t7xx_dpmaif_tx_drb_buf_init(struct dpmaif_tx_queue *txq) |
538 | { |
539 | size_t brb_skb_size, brb_pd_size; |
540 | |
541 | brb_pd_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb); |
542 | brb_skb_size = DPMAIF_DRB_LIST_LEN * sizeof(struct dpmaif_drb_skb); |
543 | |
544 | txq->drb_size_cnt = DPMAIF_DRB_LIST_LEN; |
545 | |
546 | /* For HW && AP SW */ |
547 | txq->drb_base = dma_alloc_coherent(dev: txq->dpmaif_ctrl->dev, size: brb_pd_size, |
548 | dma_handle: &txq->drb_bus_addr, GFP_KERNEL | __GFP_ZERO); |
549 | if (!txq->drb_base) |
550 | return -ENOMEM; |
551 | |
552 | /* For AP SW to record the skb information */ |
553 | txq->drb_skb_base = devm_kzalloc(dev: txq->dpmaif_ctrl->dev, size: brb_skb_size, GFP_KERNEL); |
554 | if (!txq->drb_skb_base) { |
555 | dma_free_coherent(dev: txq->dpmaif_ctrl->dev, size: brb_pd_size, |
556 | cpu_addr: txq->drb_base, dma_handle: txq->drb_bus_addr); |
557 | return -ENOMEM; |
558 | } |
559 | |
560 | return 0; |
561 | } |
562 | |
563 | static void t7xx_dpmaif_tx_free_drb_skb(struct dpmaif_tx_queue *txq) |
564 | { |
565 | struct dpmaif_drb_skb *drb_skb, *drb_skb_base = txq->drb_skb_base; |
566 | unsigned int i; |
567 | |
568 | if (!drb_skb_base) |
569 | return; |
570 | |
571 | for (i = 0; i < txq->drb_size_cnt; i++) { |
572 | drb_skb = drb_skb_base + i; |
573 | if (!drb_skb->skb) |
574 | continue; |
575 | |
576 | if (!drb_skb->is_msg) |
577 | dma_unmap_single(txq->dpmaif_ctrl->dev, drb_skb->bus_addr, |
578 | drb_skb->data_len, DMA_TO_DEVICE); |
579 | |
580 | if (drb_skb->is_last) { |
581 | dev_kfree_skb(drb_skb->skb); |
582 | drb_skb->skb = NULL; |
583 | } |
584 | } |
585 | } |
586 | |
587 | static void t7xx_dpmaif_tx_drb_buf_rel(struct dpmaif_tx_queue *txq) |
588 | { |
589 | if (txq->drb_base) |
590 | dma_free_coherent(dev: txq->dpmaif_ctrl->dev, |
591 | size: txq->drb_size_cnt * sizeof(struct dpmaif_drb), |
592 | cpu_addr: txq->drb_base, dma_handle: txq->drb_bus_addr); |
593 | |
594 | t7xx_dpmaif_tx_free_drb_skb(txq); |
595 | } |
596 | |
597 | /** |
598 | * t7xx_dpmaif_txq_init() - Initialize TX queue. |
599 | * @txq: Pointer to struct dpmaif_tx_queue. |
600 | * |
601 | * Initialize the TX queue data structure and allocate memory for it to use. |
602 | * |
603 | * Return: |
604 | * * 0 - Success. |
605 | * * -ERROR - Error code from failure sub-initializations. |
606 | */ |
607 | int t7xx_dpmaif_txq_init(struct dpmaif_tx_queue *txq) |
608 | { |
609 | int ret; |
610 | |
611 | skb_queue_head_init(list: &txq->tx_skb_head); |
612 | init_waitqueue_head(&txq->req_wq); |
613 | atomic_set(v: &txq->tx_budget, DPMAIF_DRB_LIST_LEN); |
614 | |
615 | ret = t7xx_dpmaif_tx_drb_buf_init(txq); |
616 | if (ret) { |
617 | dev_err(txq->dpmaif_ctrl->dev, "Failed to initialize DRB buffers: %d\n" , ret); |
618 | return ret; |
619 | } |
620 | |
621 | txq->worker = alloc_ordered_workqueue("md_dpmaif_tx%d_worker" , |
622 | WQ_MEM_RECLAIM | (txq->index ? 0 : WQ_HIGHPRI), |
623 | txq->index); |
624 | if (!txq->worker) |
625 | return -ENOMEM; |
626 | |
627 | INIT_WORK(&txq->dpmaif_tx_work, t7xx_dpmaif_tx_done); |
628 | spin_lock_init(&txq->tx_lock); |
629 | |
630 | return 0; |
631 | } |
632 | |
633 | void t7xx_dpmaif_txq_free(struct dpmaif_tx_queue *txq) |
634 | { |
635 | if (txq->worker) |
636 | destroy_workqueue(wq: txq->worker); |
637 | |
638 | skb_queue_purge(list: &txq->tx_skb_head); |
639 | t7xx_dpmaif_tx_drb_buf_rel(txq); |
640 | } |
641 | |
642 | void t7xx_dpmaif_tx_stop(struct dpmaif_ctrl *dpmaif_ctrl) |
643 | { |
644 | int i; |
645 | |
646 | for (i = 0; i < DPMAIF_TXQ_NUM; i++) { |
647 | struct dpmaif_tx_queue *txq; |
648 | int count = 0; |
649 | |
650 | txq = &dpmaif_ctrl->txq[i]; |
651 | txq->que_started = false; |
652 | /* Make sure TXQ is disabled */ |
653 | smp_mb(); |
654 | |
655 | /* Wait for active Tx to be done */ |
656 | while (atomic_read(v: &txq->tx_processing)) { |
657 | if (++count >= DPMAIF_MAX_CHECK_COUNT) { |
658 | dev_err(dpmaif_ctrl->dev, "TX queue stop failed\n" ); |
659 | break; |
660 | } |
661 | } |
662 | } |
663 | } |
664 | |
665 | static void t7xx_dpmaif_txq_flush_rel(struct dpmaif_tx_queue *txq) |
666 | { |
667 | txq->que_started = false; |
668 | |
669 | cancel_work_sync(work: &txq->dpmaif_tx_work); |
670 | flush_work(work: &txq->dpmaif_tx_work); |
671 | t7xx_dpmaif_tx_free_drb_skb(txq); |
672 | |
673 | txq->drb_rd_idx = 0; |
674 | txq->drb_wr_idx = 0; |
675 | txq->drb_release_rd_idx = 0; |
676 | } |
677 | |
678 | void t7xx_dpmaif_tx_clear(struct dpmaif_ctrl *dpmaif_ctrl) |
679 | { |
680 | int i; |
681 | |
682 | for (i = 0; i < DPMAIF_TXQ_NUM; i++) |
683 | t7xx_dpmaif_txq_flush_rel(txq: &dpmaif_ctrl->txq[i]); |
684 | } |
685 | |