1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2021, MediaTek Inc. |
4 | * Copyright (c) 2021-2022, Intel Corporation. |
5 | * |
6 | * Authors: |
7 | * Amir Hanania <amir.hanania@intel.com> |
8 | * Haijun Liu <haijun.liu@mediatek.com> |
9 | * Eliot Lee <eliot.lee@intel.com> |
10 | * Moises Veleta <moises.veleta@intel.com> |
11 | * Ricardo Martinez <ricardo.martinez@linux.intel.com> |
12 | * |
13 | * Contributors: |
14 | * Andy Shevchenko <andriy.shevchenko@linux.intel.com> |
15 | * Chiranjeevi Rapolu <chiranjeevi.rapolu@intel.com> |
16 | * Sreehari Kancharla <sreehari.kancharla@intel.com> |
17 | */ |
18 | |
19 | #include <linux/atomic.h> |
20 | #include <linux/bitfield.h> |
21 | #include <linux/bitops.h> |
22 | #include <linux/device.h> |
23 | #include <linux/dma-direction.h> |
24 | #include <linux/dma-mapping.h> |
25 | #include <linux/gfp.h> |
26 | #include <linux/err.h> |
27 | #include <linux/iopoll.h> |
28 | #include <linux/jiffies.h> |
29 | #include <linux/kernel.h> |
30 | #include <linux/kthread.h> |
31 | #include <linux/list.h> |
32 | #include <linux/minmax.h> |
33 | #include <linux/mm.h> |
34 | #include <linux/netdevice.h> |
35 | #include <linux/pm_runtime.h> |
36 | #include <linux/sched.h> |
37 | #include <linux/skbuff.h> |
38 | #include <linux/slab.h> |
39 | #include <linux/spinlock.h> |
40 | #include <linux/string.h> |
41 | #include <linux/types.h> |
42 | #include <linux/wait.h> |
43 | #include <linux/workqueue.h> |
44 | |
45 | #include "t7xx_dpmaif.h" |
46 | #include "t7xx_hif_dpmaif.h" |
47 | #include "t7xx_hif_dpmaif_rx.h" |
48 | #include "t7xx_netdev.h" |
49 | #include "t7xx_pci.h" |
50 | |
51 | #define DPMAIF_BAT_COUNT 8192 |
52 | #define DPMAIF_FRG_COUNT 4814 |
53 | #define DPMAIF_PIT_COUNT (DPMAIF_BAT_COUNT * 2) |
54 | |
55 | #define DPMAIF_BAT_CNT_THRESHOLD 30 |
56 | #define DPMAIF_PIT_CNT_THRESHOLD 60 |
57 | #define DPMAIF_RX_PUSH_THRESHOLD_MASK GENMASK(2, 0) |
58 | #define DPMAIF_NOTIFY_RELEASE_COUNT 128 |
59 | #define DPMAIF_POLL_PIT_TIME_US 20 |
60 | #define DPMAIF_POLL_PIT_MAX_TIME_US 2000 |
61 | #define DPMAIF_WQ_TIME_LIMIT_MS 2 |
62 | #define DPMAIF_CS_RESULT_PASS 0 |
63 | |
64 | /* Packet type */ |
65 | #define DES_PT_PD 0 |
66 | #define DES_PT_MSG 1 |
67 | /* Buffer type */ |
68 | #define PKT_BUF_FRAG 1 |
69 | |
70 | static unsigned int t7xx_normal_pit_bid(const struct dpmaif_pit *pit_info) |
71 | { |
72 | u32 value; |
73 | |
74 | value = FIELD_GET(PD_PIT_H_BID, le32_to_cpu(pit_info->pd.footer)); |
75 | value <<= 13; |
76 | value += FIELD_GET(PD_PIT_BUFFER_ID, le32_to_cpu(pit_info->header)); |
77 | return value; |
78 | } |
79 | |
80 | static int t7xx_dpmaif_update_bat_wr_idx(struct dpmaif_ctrl *dpmaif_ctrl, |
81 | const unsigned int q_num, const unsigned int bat_cnt) |
82 | { |
83 | struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num]; |
84 | struct dpmaif_bat_request *bat_req = rxq->bat_req; |
85 | unsigned int old_rl_idx, new_wr_idx, old_wr_idx; |
86 | |
87 | if (!rxq->que_started) { |
88 | dev_err(dpmaif_ctrl->dev, "RX queue %d has not been started\n" , rxq->index); |
89 | return -EINVAL; |
90 | } |
91 | |
92 | old_rl_idx = bat_req->bat_release_rd_idx; |
93 | old_wr_idx = bat_req->bat_wr_idx; |
94 | new_wr_idx = old_wr_idx + bat_cnt; |
95 | |
96 | if (old_rl_idx > old_wr_idx && new_wr_idx >= old_rl_idx) |
97 | goto err_flow; |
98 | |
99 | if (new_wr_idx >= bat_req->bat_size_cnt) { |
100 | new_wr_idx -= bat_req->bat_size_cnt; |
101 | if (new_wr_idx >= old_rl_idx) |
102 | goto err_flow; |
103 | } |
104 | |
105 | bat_req->bat_wr_idx = new_wr_idx; |
106 | return 0; |
107 | |
108 | err_flow: |
109 | dev_err(dpmaif_ctrl->dev, "RX BAT flow check fail\n" ); |
110 | return -EINVAL; |
111 | } |
112 | |
113 | static bool t7xx_alloc_and_map_skb_info(const struct dpmaif_ctrl *dpmaif_ctrl, |
114 | const unsigned int size, struct dpmaif_bat_skb *cur_skb) |
115 | { |
116 | dma_addr_t data_bus_addr; |
117 | struct sk_buff *skb; |
118 | |
119 | skb = __dev_alloc_skb(length: size, GFP_KERNEL); |
120 | if (!skb) |
121 | return false; |
122 | |
123 | data_bus_addr = dma_map_single(dpmaif_ctrl->dev, skb->data, size, DMA_FROM_DEVICE); |
124 | if (dma_mapping_error(dev: dpmaif_ctrl->dev, dma_addr: data_bus_addr)) { |
125 | dev_err_ratelimited(dpmaif_ctrl->dev, "DMA mapping error\n" ); |
126 | dev_kfree_skb_any(skb); |
127 | return false; |
128 | } |
129 | |
130 | cur_skb->skb = skb; |
131 | cur_skb->data_bus_addr = data_bus_addr; |
132 | cur_skb->data_len = size; |
133 | |
134 | return true; |
135 | } |
136 | |
137 | static void t7xx_unmap_bat_skb(struct device *dev, struct dpmaif_bat_skb *bat_skb_base, |
138 | unsigned int index) |
139 | { |
140 | struct dpmaif_bat_skb *bat_skb = bat_skb_base + index; |
141 | |
142 | if (bat_skb->skb) { |
143 | dma_unmap_single(dev, bat_skb->data_bus_addr, bat_skb->data_len, DMA_FROM_DEVICE); |
144 | dev_kfree_skb(bat_skb->skb); |
145 | bat_skb->skb = NULL; |
146 | } |
147 | } |
148 | |
149 | /** |
150 | * t7xx_dpmaif_rx_buf_alloc() - Allocate buffers for the BAT ring. |
151 | * @dpmaif_ctrl: Pointer to DPMAIF context structure. |
152 | * @bat_req: Pointer to BAT request structure. |
153 | * @q_num: Queue number. |
154 | * @buf_cnt: Number of buffers to allocate. |
155 | * @initial: Indicates if the ring is being populated for the first time. |
156 | * |
157 | * Allocate skb and store the start address of the data buffer into the BAT ring. |
158 | * If this is not the initial call, notify the HW about the new entries. |
159 | * |
160 | * Return: |
161 | * * 0 - Success. |
162 | * * -ERROR - Error code. |
163 | */ |
164 | int t7xx_dpmaif_rx_buf_alloc(struct dpmaif_ctrl *dpmaif_ctrl, |
165 | const struct dpmaif_bat_request *bat_req, |
166 | const unsigned int q_num, const unsigned int buf_cnt, |
167 | const bool initial) |
168 | { |
169 | unsigned int i, bat_cnt, bat_max_cnt, bat_start_idx; |
170 | int ret; |
171 | |
172 | if (!buf_cnt || buf_cnt > bat_req->bat_size_cnt) |
173 | return -EINVAL; |
174 | |
175 | /* Check BAT buffer space */ |
176 | bat_max_cnt = bat_req->bat_size_cnt; |
177 | |
178 | bat_cnt = t7xx_ring_buf_rd_wr_count(total_cnt: bat_max_cnt, rd_idx: bat_req->bat_release_rd_idx, |
179 | wr_idx: bat_req->bat_wr_idx, DPMAIF_WRITE); |
180 | if (buf_cnt > bat_cnt) |
181 | return -ENOMEM; |
182 | |
183 | bat_start_idx = bat_req->bat_wr_idx; |
184 | |
185 | for (i = 0; i < buf_cnt; i++) { |
186 | unsigned int cur_bat_idx = bat_start_idx + i; |
187 | struct dpmaif_bat_skb *cur_skb; |
188 | struct dpmaif_bat *cur_bat; |
189 | |
190 | if (cur_bat_idx >= bat_max_cnt) |
191 | cur_bat_idx -= bat_max_cnt; |
192 | |
193 | cur_skb = (struct dpmaif_bat_skb *)bat_req->bat_skb + cur_bat_idx; |
194 | if (!cur_skb->skb && |
195 | !t7xx_alloc_and_map_skb_info(dpmaif_ctrl, size: bat_req->pkt_buf_sz, cur_skb)) |
196 | break; |
197 | |
198 | cur_bat = (struct dpmaif_bat *)bat_req->bat_base + cur_bat_idx; |
199 | cur_bat->buffer_addr_ext = upper_32_bits(cur_skb->data_bus_addr); |
200 | cur_bat->p_buffer_addr = lower_32_bits(cur_skb->data_bus_addr); |
201 | } |
202 | |
203 | if (!i) |
204 | return -ENOMEM; |
205 | |
206 | ret = t7xx_dpmaif_update_bat_wr_idx(dpmaif_ctrl, q_num, bat_cnt: i); |
207 | if (ret) |
208 | goto err_unmap_skbs; |
209 | |
210 | if (!initial) { |
211 | unsigned int hw_wr_idx; |
212 | |
213 | ret = t7xx_dpmaif_dl_snd_hw_bat_cnt(hw_info: &dpmaif_ctrl->hw_info, bat_entry_cnt: i); |
214 | if (ret) |
215 | goto err_unmap_skbs; |
216 | |
217 | hw_wr_idx = t7xx_dpmaif_dl_get_bat_wr_idx(hw_info: &dpmaif_ctrl->hw_info, |
218 | DPF_RX_QNO_DFT); |
219 | if (hw_wr_idx != bat_req->bat_wr_idx) { |
220 | ret = -EFAULT; |
221 | dev_err(dpmaif_ctrl->dev, "Write index mismatch in RX ring\n" ); |
222 | goto err_unmap_skbs; |
223 | } |
224 | } |
225 | |
226 | return 0; |
227 | |
228 | err_unmap_skbs: |
229 | while (--i > 0) |
230 | t7xx_unmap_bat_skb(dev: dpmaif_ctrl->dev, bat_skb_base: bat_req->bat_skb, index: i); |
231 | |
232 | return ret; |
233 | } |
234 | |
235 | static int t7xx_dpmaifq_release_pit_entry(struct dpmaif_rx_queue *rxq, |
236 | const unsigned int rel_entry_num) |
237 | { |
238 | struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info; |
239 | unsigned int old_rel_idx, new_rel_idx, hw_wr_idx; |
240 | int ret; |
241 | |
242 | if (!rxq->que_started) |
243 | return 0; |
244 | |
245 | if (rel_entry_num >= rxq->pit_size_cnt) { |
246 | dev_err(rxq->dpmaif_ctrl->dev, "Invalid PIT release index\n" ); |
247 | return -EINVAL; |
248 | } |
249 | |
250 | old_rel_idx = rxq->pit_release_rd_idx; |
251 | new_rel_idx = old_rel_idx + rel_entry_num; |
252 | hw_wr_idx = rxq->pit_wr_idx; |
253 | if (hw_wr_idx < old_rel_idx && new_rel_idx >= rxq->pit_size_cnt) |
254 | new_rel_idx -= rxq->pit_size_cnt; |
255 | |
256 | ret = t7xx_dpmaif_dlq_add_pit_remain_cnt(hw_info, dlq_pit_idx: rxq->index, pit_remain_cnt: rel_entry_num); |
257 | if (ret) { |
258 | dev_err(rxq->dpmaif_ctrl->dev, "PIT release failure: %d\n" , ret); |
259 | return ret; |
260 | } |
261 | |
262 | rxq->pit_release_rd_idx = new_rel_idx; |
263 | return 0; |
264 | } |
265 | |
266 | static void t7xx_dpmaif_set_bat_mask(struct dpmaif_bat_request *bat_req, unsigned int idx) |
267 | { |
268 | unsigned long flags; |
269 | |
270 | spin_lock_irqsave(&bat_req->mask_lock, flags); |
271 | set_bit(nr: idx, addr: bat_req->bat_bitmap); |
272 | spin_unlock_irqrestore(lock: &bat_req->mask_lock, flags); |
273 | } |
274 | |
275 | static int t7xx_frag_bat_cur_bid_check(struct dpmaif_rx_queue *rxq, |
276 | const unsigned int cur_bid) |
277 | { |
278 | struct dpmaif_bat_request *bat_frag = rxq->bat_frag; |
279 | struct dpmaif_bat_page *bat_page; |
280 | |
281 | if (cur_bid >= DPMAIF_FRG_COUNT) |
282 | return -EINVAL; |
283 | |
284 | bat_page = bat_frag->bat_skb + cur_bid; |
285 | if (!bat_page->page) |
286 | return -EINVAL; |
287 | |
288 | return 0; |
289 | } |
290 | |
291 | static void t7xx_unmap_bat_page(struct device *dev, struct dpmaif_bat_page *bat_page_base, |
292 | unsigned int index) |
293 | { |
294 | struct dpmaif_bat_page *bat_page = bat_page_base + index; |
295 | |
296 | if (bat_page->page) { |
297 | dma_unmap_page(dev, bat_page->data_bus_addr, bat_page->data_len, DMA_FROM_DEVICE); |
298 | put_page(page: bat_page->page); |
299 | bat_page->page = NULL; |
300 | } |
301 | } |
302 | |
303 | /** |
304 | * t7xx_dpmaif_rx_frag_alloc() - Allocates buffers for the Fragment BAT ring. |
305 | * @dpmaif_ctrl: Pointer to DPMAIF context structure. |
306 | * @bat_req: Pointer to BAT request structure. |
307 | * @buf_cnt: Number of buffers to allocate. |
308 | * @initial: Indicates if the ring is being populated for the first time. |
309 | * |
310 | * Fragment BAT is used when the received packet does not fit in a normal BAT entry. |
311 | * This function allocates a page fragment and stores the start address of the page |
312 | * into the Fragment BAT ring. |
313 | * If this is not the initial call, notify the HW about the new entries. |
314 | * |
315 | * Return: |
316 | * * 0 - Success. |
317 | * * -ERROR - Error code. |
318 | */ |
319 | int t7xx_dpmaif_rx_frag_alloc(struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req, |
320 | const unsigned int buf_cnt, const bool initial) |
321 | { |
322 | unsigned int buf_space, cur_bat_idx = bat_req->bat_wr_idx; |
323 | struct dpmaif_bat_page *bat_skb = bat_req->bat_skb; |
324 | int ret = 0, i; |
325 | |
326 | if (!buf_cnt || buf_cnt > bat_req->bat_size_cnt) |
327 | return -EINVAL; |
328 | |
329 | buf_space = t7xx_ring_buf_rd_wr_count(total_cnt: bat_req->bat_size_cnt, |
330 | rd_idx: bat_req->bat_release_rd_idx, wr_idx: bat_req->bat_wr_idx, |
331 | DPMAIF_WRITE); |
332 | if (buf_cnt > buf_space) { |
333 | dev_err(dpmaif_ctrl->dev, |
334 | "Requested more buffers than the space available in RX frag ring\n" ); |
335 | return -EINVAL; |
336 | } |
337 | |
338 | for (i = 0; i < buf_cnt; i++) { |
339 | struct dpmaif_bat_page *cur_page = bat_skb + cur_bat_idx; |
340 | struct dpmaif_bat *cur_bat; |
341 | dma_addr_t data_base_addr; |
342 | |
343 | if (!cur_page->page) { |
344 | unsigned long offset; |
345 | struct page *page; |
346 | void *data; |
347 | |
348 | data = netdev_alloc_frag(fragsz: bat_req->pkt_buf_sz); |
349 | if (!data) |
350 | break; |
351 | |
352 | page = virt_to_head_page(x: data); |
353 | offset = data - page_address(page); |
354 | |
355 | data_base_addr = dma_map_page(dpmaif_ctrl->dev, page, offset, |
356 | bat_req->pkt_buf_sz, DMA_FROM_DEVICE); |
357 | if (dma_mapping_error(dev: dpmaif_ctrl->dev, dma_addr: data_base_addr)) { |
358 | put_page(page: virt_to_head_page(x: data)); |
359 | dev_err(dpmaif_ctrl->dev, "DMA mapping fail\n" ); |
360 | break; |
361 | } |
362 | |
363 | cur_page->page = page; |
364 | cur_page->data_bus_addr = data_base_addr; |
365 | cur_page->offset = offset; |
366 | cur_page->data_len = bat_req->pkt_buf_sz; |
367 | } |
368 | |
369 | data_base_addr = cur_page->data_bus_addr; |
370 | cur_bat = (struct dpmaif_bat *)bat_req->bat_base + cur_bat_idx; |
371 | cur_bat->buffer_addr_ext = upper_32_bits(data_base_addr); |
372 | cur_bat->p_buffer_addr = lower_32_bits(data_base_addr); |
373 | cur_bat_idx = t7xx_ring_buf_get_next_wr_idx(buf_len: bat_req->bat_size_cnt, buf_idx: cur_bat_idx); |
374 | } |
375 | |
376 | bat_req->bat_wr_idx = cur_bat_idx; |
377 | |
378 | if (!initial) |
379 | t7xx_dpmaif_dl_snd_hw_frg_cnt(hw_info: &dpmaif_ctrl->hw_info, frg_entry_cnt: i); |
380 | |
381 | if (i < buf_cnt) { |
382 | ret = -ENOMEM; |
383 | if (initial) { |
384 | while (--i > 0) |
385 | t7xx_unmap_bat_page(dev: dpmaif_ctrl->dev, bat_page_base: bat_req->bat_skb, index: i); |
386 | } |
387 | } |
388 | |
389 | return ret; |
390 | } |
391 | |
392 | static int t7xx_dpmaif_set_frag_to_skb(const struct dpmaif_rx_queue *rxq, |
393 | const struct dpmaif_pit *pkt_info, |
394 | struct sk_buff *skb) |
395 | { |
396 | unsigned long long data_bus_addr, data_base_addr; |
397 | struct device *dev = rxq->dpmaif_ctrl->dev; |
398 | struct dpmaif_bat_page *page_info; |
399 | unsigned int data_len; |
400 | int data_offset; |
401 | |
402 | page_info = rxq->bat_frag->bat_skb; |
403 | page_info += t7xx_normal_pit_bid(pit_info: pkt_info); |
404 | dma_unmap_page(dev, page_info->data_bus_addr, page_info->data_len, DMA_FROM_DEVICE); |
405 | |
406 | if (!page_info->page) |
407 | return -EINVAL; |
408 | |
409 | data_bus_addr = le32_to_cpu(pkt_info->pd.data_addr_h); |
410 | data_bus_addr = (data_bus_addr << 32) + le32_to_cpu(pkt_info->pd.data_addr_l); |
411 | data_base_addr = page_info->data_bus_addr; |
412 | data_offset = data_bus_addr - data_base_addr; |
413 | data_offset += page_info->offset; |
414 | data_len = FIELD_GET(PD_PIT_DATA_LEN, le32_to_cpu(pkt_info->header)); |
415 | skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page: page_info->page, |
416 | off: data_offset, size: data_len, truesize: page_info->data_len); |
417 | |
418 | page_info->page = NULL; |
419 | page_info->offset = 0; |
420 | page_info->data_len = 0; |
421 | return 0; |
422 | } |
423 | |
424 | static int t7xx_dpmaif_get_frag(struct dpmaif_rx_queue *rxq, |
425 | const struct dpmaif_pit *pkt_info, |
426 | const struct dpmaif_cur_rx_skb_info *skb_info) |
427 | { |
428 | unsigned int cur_bid = t7xx_normal_pit_bid(pit_info: pkt_info); |
429 | int ret; |
430 | |
431 | ret = t7xx_frag_bat_cur_bid_check(rxq, cur_bid); |
432 | if (ret < 0) |
433 | return ret; |
434 | |
435 | ret = t7xx_dpmaif_set_frag_to_skb(rxq, pkt_info, skb: skb_info->cur_skb); |
436 | if (ret < 0) { |
437 | dev_err(rxq->dpmaif_ctrl->dev, "Failed to set frag data to skb: %d\n" , ret); |
438 | return ret; |
439 | } |
440 | |
441 | t7xx_dpmaif_set_bat_mask(bat_req: rxq->bat_frag, idx: cur_bid); |
442 | return 0; |
443 | } |
444 | |
445 | static int t7xx_bat_cur_bid_check(struct dpmaif_rx_queue *rxq, const unsigned int cur_bid) |
446 | { |
447 | struct dpmaif_bat_skb *bat_skb = rxq->bat_req->bat_skb; |
448 | |
449 | bat_skb += cur_bid; |
450 | if (cur_bid >= DPMAIF_BAT_COUNT || !bat_skb->skb) |
451 | return -EINVAL; |
452 | |
453 | return 0; |
454 | } |
455 | |
456 | static int t7xx_dpmaif_read_pit_seq(const struct dpmaif_pit *pit) |
457 | { |
458 | return FIELD_GET(PD_PIT_PIT_SEQ, le32_to_cpu(pit->pd.footer)); |
459 | } |
460 | |
461 | static int t7xx_dpmaif_check_pit_seq(struct dpmaif_rx_queue *rxq, |
462 | const struct dpmaif_pit *pit) |
463 | { |
464 | unsigned int cur_pit_seq, expect_pit_seq = rxq->expect_pit_seq; |
465 | |
466 | if (read_poll_timeout_atomic(t7xx_dpmaif_read_pit_seq, cur_pit_seq, |
467 | cur_pit_seq == expect_pit_seq, DPMAIF_POLL_PIT_TIME_US, |
468 | DPMAIF_POLL_PIT_MAX_TIME_US, false, pit)) |
469 | return -EFAULT; |
470 | |
471 | rxq->expect_pit_seq++; |
472 | if (rxq->expect_pit_seq >= DPMAIF_DL_PIT_SEQ_VALUE) |
473 | rxq->expect_pit_seq = 0; |
474 | |
475 | return 0; |
476 | } |
477 | |
478 | static unsigned int t7xx_dpmaif_avail_pkt_bat_cnt(struct dpmaif_bat_request *bat_req) |
479 | { |
480 | unsigned int zero_index; |
481 | unsigned long flags; |
482 | |
483 | spin_lock_irqsave(&bat_req->mask_lock, flags); |
484 | |
485 | zero_index = find_next_zero_bit(addr: bat_req->bat_bitmap, size: bat_req->bat_size_cnt, |
486 | offset: bat_req->bat_release_rd_idx); |
487 | |
488 | if (zero_index < bat_req->bat_size_cnt) { |
489 | spin_unlock_irqrestore(lock: &bat_req->mask_lock, flags); |
490 | return zero_index - bat_req->bat_release_rd_idx; |
491 | } |
492 | |
493 | /* limiting the search till bat_release_rd_idx */ |
494 | zero_index = find_first_zero_bit(addr: bat_req->bat_bitmap, size: bat_req->bat_release_rd_idx); |
495 | spin_unlock_irqrestore(lock: &bat_req->mask_lock, flags); |
496 | return bat_req->bat_size_cnt - bat_req->bat_release_rd_idx + zero_index; |
497 | } |
498 | |
499 | static int t7xx_dpmaif_release_bat_entry(const struct dpmaif_rx_queue *rxq, |
500 | const unsigned int rel_entry_num, |
501 | const enum bat_type buf_type) |
502 | { |
503 | struct dpmaif_hw_info *hw_info = &rxq->dpmaif_ctrl->hw_info; |
504 | unsigned int old_rel_idx, new_rel_idx, hw_rd_idx, i; |
505 | struct dpmaif_bat_request *bat; |
506 | unsigned long flags; |
507 | |
508 | if (!rxq->que_started || !rel_entry_num) |
509 | return -EINVAL; |
510 | |
511 | if (buf_type == BAT_TYPE_FRAG) { |
512 | bat = rxq->bat_frag; |
513 | hw_rd_idx = t7xx_dpmaif_dl_get_frg_rd_idx(hw_info, q_num: rxq->index); |
514 | } else { |
515 | bat = rxq->bat_req; |
516 | hw_rd_idx = t7xx_dpmaif_dl_get_bat_rd_idx(hw_info, q_num: rxq->index); |
517 | } |
518 | |
519 | if (rel_entry_num >= bat->bat_size_cnt) |
520 | return -EINVAL; |
521 | |
522 | old_rel_idx = bat->bat_release_rd_idx; |
523 | new_rel_idx = old_rel_idx + rel_entry_num; |
524 | |
525 | /* Do not need to release if the queue is empty */ |
526 | if (bat->bat_wr_idx == old_rel_idx) |
527 | return 0; |
528 | |
529 | if (hw_rd_idx >= old_rel_idx) { |
530 | if (new_rel_idx > hw_rd_idx) |
531 | return -EINVAL; |
532 | } |
533 | |
534 | if (new_rel_idx >= bat->bat_size_cnt) { |
535 | new_rel_idx -= bat->bat_size_cnt; |
536 | if (new_rel_idx > hw_rd_idx) |
537 | return -EINVAL; |
538 | } |
539 | |
540 | spin_lock_irqsave(&bat->mask_lock, flags); |
541 | for (i = 0; i < rel_entry_num; i++) { |
542 | unsigned int index = bat->bat_release_rd_idx + i; |
543 | |
544 | if (index >= bat->bat_size_cnt) |
545 | index -= bat->bat_size_cnt; |
546 | |
547 | clear_bit(nr: index, addr: bat->bat_bitmap); |
548 | } |
549 | spin_unlock_irqrestore(lock: &bat->mask_lock, flags); |
550 | |
551 | bat->bat_release_rd_idx = new_rel_idx; |
552 | return rel_entry_num; |
553 | } |
554 | |
555 | static int t7xx_dpmaif_pit_release_and_add(struct dpmaif_rx_queue *rxq) |
556 | { |
557 | int ret; |
558 | |
559 | if (rxq->pit_remain_release_cnt < DPMAIF_PIT_CNT_THRESHOLD) |
560 | return 0; |
561 | |
562 | ret = t7xx_dpmaifq_release_pit_entry(rxq, rel_entry_num: rxq->pit_remain_release_cnt); |
563 | if (ret) |
564 | return ret; |
565 | |
566 | rxq->pit_remain_release_cnt = 0; |
567 | return 0; |
568 | } |
569 | |
570 | static int t7xx_dpmaif_bat_release_and_add(const struct dpmaif_rx_queue *rxq) |
571 | { |
572 | unsigned int bid_cnt; |
573 | int ret; |
574 | |
575 | bid_cnt = t7xx_dpmaif_avail_pkt_bat_cnt(bat_req: rxq->bat_req); |
576 | if (bid_cnt < DPMAIF_BAT_CNT_THRESHOLD) |
577 | return 0; |
578 | |
579 | ret = t7xx_dpmaif_release_bat_entry(rxq, rel_entry_num: bid_cnt, buf_type: BAT_TYPE_NORMAL); |
580 | if (ret <= 0) { |
581 | dev_err(rxq->dpmaif_ctrl->dev, "Release PKT BAT failed: %d\n" , ret); |
582 | return ret; |
583 | } |
584 | |
585 | ret = t7xx_dpmaif_rx_buf_alloc(dpmaif_ctrl: rxq->dpmaif_ctrl, bat_req: rxq->bat_req, q_num: rxq->index, buf_cnt: bid_cnt, initial: false); |
586 | if (ret < 0) |
587 | dev_err(rxq->dpmaif_ctrl->dev, "Allocate new RX buffer failed: %d\n" , ret); |
588 | |
589 | return ret; |
590 | } |
591 | |
592 | static int t7xx_dpmaif_frag_bat_release_and_add(const struct dpmaif_rx_queue *rxq) |
593 | { |
594 | unsigned int bid_cnt; |
595 | int ret; |
596 | |
597 | bid_cnt = t7xx_dpmaif_avail_pkt_bat_cnt(bat_req: rxq->bat_frag); |
598 | if (bid_cnt < DPMAIF_BAT_CNT_THRESHOLD) |
599 | return 0; |
600 | |
601 | ret = t7xx_dpmaif_release_bat_entry(rxq, rel_entry_num: bid_cnt, buf_type: BAT_TYPE_FRAG); |
602 | if (ret <= 0) { |
603 | dev_err(rxq->dpmaif_ctrl->dev, "Release BAT entry failed: %d\n" , ret); |
604 | return ret; |
605 | } |
606 | |
607 | return t7xx_dpmaif_rx_frag_alloc(dpmaif_ctrl: rxq->dpmaif_ctrl, bat_req: rxq->bat_frag, buf_cnt: bid_cnt, initial: false); |
608 | } |
609 | |
610 | static void t7xx_dpmaif_parse_msg_pit(const struct dpmaif_rx_queue *rxq, |
611 | const struct dpmaif_pit *msg_pit, |
612 | struct dpmaif_cur_rx_skb_info *skb_info) |
613 | { |
614 | int = le32_to_cpu(msg_pit->header); |
615 | |
616 | skb_info->cur_chn_idx = FIELD_GET(MSG_PIT_CHANNEL_ID, header); |
617 | skb_info->check_sum = FIELD_GET(MSG_PIT_CHECKSUM, header); |
618 | skb_info->pit_dp = FIELD_GET(MSG_PIT_DP, header); |
619 | skb_info->pkt_type = FIELD_GET(MSG_PIT_IP, le32_to_cpu(msg_pit->msg.params_3)); |
620 | } |
621 | |
622 | static int t7xx_dpmaif_set_data_to_skb(const struct dpmaif_rx_queue *rxq, |
623 | const struct dpmaif_pit *pkt_info, |
624 | struct dpmaif_cur_rx_skb_info *skb_info) |
625 | { |
626 | unsigned long long data_bus_addr, data_base_addr; |
627 | struct device *dev = rxq->dpmaif_ctrl->dev; |
628 | struct dpmaif_bat_skb *bat_skb; |
629 | unsigned int data_len; |
630 | struct sk_buff *skb; |
631 | int data_offset; |
632 | |
633 | bat_skb = rxq->bat_req->bat_skb; |
634 | bat_skb += t7xx_normal_pit_bid(pit_info: pkt_info); |
635 | dma_unmap_single(dev, bat_skb->data_bus_addr, bat_skb->data_len, DMA_FROM_DEVICE); |
636 | |
637 | data_bus_addr = le32_to_cpu(pkt_info->pd.data_addr_h); |
638 | data_bus_addr = (data_bus_addr << 32) + le32_to_cpu(pkt_info->pd.data_addr_l); |
639 | data_base_addr = bat_skb->data_bus_addr; |
640 | data_offset = data_bus_addr - data_base_addr; |
641 | data_len = FIELD_GET(PD_PIT_DATA_LEN, le32_to_cpu(pkt_info->header)); |
642 | skb = bat_skb->skb; |
643 | skb->len = 0; |
644 | skb_reset_tail_pointer(skb); |
645 | skb_reserve(skb, len: data_offset); |
646 | |
647 | if (skb->tail + data_len > skb->end) { |
648 | dev_err(dev, "No buffer space available\n" ); |
649 | return -ENOBUFS; |
650 | } |
651 | |
652 | skb_put(skb, len: data_len); |
653 | skb_info->cur_skb = skb; |
654 | bat_skb->skb = NULL; |
655 | return 0; |
656 | } |
657 | |
658 | static int t7xx_dpmaif_get_rx_pkt(struct dpmaif_rx_queue *rxq, |
659 | const struct dpmaif_pit *pkt_info, |
660 | struct dpmaif_cur_rx_skb_info *skb_info) |
661 | { |
662 | unsigned int cur_bid = t7xx_normal_pit_bid(pit_info: pkt_info); |
663 | int ret; |
664 | |
665 | ret = t7xx_bat_cur_bid_check(rxq, cur_bid); |
666 | if (ret < 0) |
667 | return ret; |
668 | |
669 | ret = t7xx_dpmaif_set_data_to_skb(rxq, pkt_info, skb_info); |
670 | if (ret < 0) { |
671 | dev_err(rxq->dpmaif_ctrl->dev, "RX set data to skb failed: %d\n" , ret); |
672 | return ret; |
673 | } |
674 | |
675 | t7xx_dpmaif_set_bat_mask(bat_req: rxq->bat_req, idx: cur_bid); |
676 | return 0; |
677 | } |
678 | |
679 | static int t7xx_dpmaifq_rx_notify_hw(struct dpmaif_rx_queue *rxq) |
680 | { |
681 | struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl; |
682 | int ret; |
683 | |
684 | queue_work(wq: dpmaif_ctrl->bat_release_wq, work: &dpmaif_ctrl->bat_release_work); |
685 | |
686 | ret = t7xx_dpmaif_pit_release_and_add(rxq); |
687 | if (ret < 0) |
688 | dev_err(dpmaif_ctrl->dev, "RXQ%u update PIT failed: %d\n" , rxq->index, ret); |
689 | |
690 | return ret; |
691 | } |
692 | |
693 | static void t7xx_dpmaif_rx_skb(struct dpmaif_rx_queue *rxq, |
694 | struct dpmaif_cur_rx_skb_info *skb_info) |
695 | { |
696 | struct dpmaif_ctrl *dpmaif_ctrl = rxq->dpmaif_ctrl; |
697 | struct sk_buff *skb = skb_info->cur_skb; |
698 | struct t7xx_skb_cb *skb_cb; |
699 | u8 netif_id; |
700 | |
701 | skb_info->cur_skb = NULL; |
702 | |
703 | if (skb_info->pit_dp) { |
704 | dev_kfree_skb_any(skb); |
705 | return; |
706 | } |
707 | |
708 | skb->ip_summed = skb_info->check_sum == DPMAIF_CS_RESULT_PASS ? CHECKSUM_UNNECESSARY : |
709 | CHECKSUM_NONE; |
710 | netif_id = FIELD_GET(NETIF_MASK, skb_info->cur_chn_idx); |
711 | skb_cb = T7XX_SKB_CB(skb); |
712 | skb_cb->netif_idx = netif_id; |
713 | skb_cb->rx_pkt_type = skb_info->pkt_type; |
714 | dpmaif_ctrl->callbacks->recv_skb(dpmaif_ctrl->t7xx_dev->ccmni_ctlb, skb, &rxq->napi); |
715 | } |
716 | |
717 | static int t7xx_dpmaif_rx_start(struct dpmaif_rx_queue *rxq, const unsigned int pit_cnt, |
718 | const unsigned int budget, int *once_more) |
719 | { |
720 | unsigned int cur_pit, pit_len, rx_cnt, recv_skb_cnt = 0; |
721 | struct device *dev = rxq->dpmaif_ctrl->dev; |
722 | struct dpmaif_cur_rx_skb_info *skb_info; |
723 | int ret = 0; |
724 | |
725 | pit_len = rxq->pit_size_cnt; |
726 | skb_info = &rxq->rx_data_info; |
727 | cur_pit = rxq->pit_rd_idx; |
728 | |
729 | for (rx_cnt = 0; rx_cnt < pit_cnt; rx_cnt++) { |
730 | struct dpmaif_pit *pkt_info; |
731 | u32 val; |
732 | |
733 | if (!skb_info->msg_pit_received && recv_skb_cnt >= budget) |
734 | break; |
735 | |
736 | pkt_info = (struct dpmaif_pit *)rxq->pit_base + cur_pit; |
737 | if (t7xx_dpmaif_check_pit_seq(rxq, pit: pkt_info)) { |
738 | dev_err_ratelimited(dev, "RXQ%u checks PIT SEQ fail\n" , rxq->index); |
739 | *once_more = 1; |
740 | return recv_skb_cnt; |
741 | } |
742 | |
743 | val = FIELD_GET(PD_PIT_PACKET_TYPE, le32_to_cpu(pkt_info->header)); |
744 | if (val == DES_PT_MSG) { |
745 | if (skb_info->msg_pit_received) |
746 | dev_err(dev, "RXQ%u received repeated PIT\n" , rxq->index); |
747 | |
748 | skb_info->msg_pit_received = true; |
749 | t7xx_dpmaif_parse_msg_pit(rxq, msg_pit: pkt_info, skb_info); |
750 | } else { /* DES_PT_PD */ |
751 | val = FIELD_GET(PD_PIT_BUFFER_TYPE, le32_to_cpu(pkt_info->header)); |
752 | if (val != PKT_BUF_FRAG) |
753 | ret = t7xx_dpmaif_get_rx_pkt(rxq, pkt_info, skb_info); |
754 | else if (!skb_info->cur_skb) |
755 | ret = -EINVAL; |
756 | else |
757 | ret = t7xx_dpmaif_get_frag(rxq, pkt_info, skb_info); |
758 | |
759 | if (ret < 0) { |
760 | skb_info->err_payload = 1; |
761 | dev_err_ratelimited(dev, "RXQ%u error payload\n" , rxq->index); |
762 | } |
763 | |
764 | val = FIELD_GET(PD_PIT_CONT, le32_to_cpu(pkt_info->header)); |
765 | if (!val) { |
766 | if (!skb_info->err_payload) { |
767 | t7xx_dpmaif_rx_skb(rxq, skb_info); |
768 | } else if (skb_info->cur_skb) { |
769 | dev_kfree_skb_any(skb: skb_info->cur_skb); |
770 | skb_info->cur_skb = NULL; |
771 | } |
772 | |
773 | memset(skb_info, 0, sizeof(*skb_info)); |
774 | recv_skb_cnt++; |
775 | } |
776 | } |
777 | |
778 | cur_pit = t7xx_ring_buf_get_next_wr_idx(buf_len: pit_len, buf_idx: cur_pit); |
779 | rxq->pit_rd_idx = cur_pit; |
780 | rxq->pit_remain_release_cnt++; |
781 | |
782 | if (rx_cnt > 0 && !(rx_cnt % DPMAIF_NOTIFY_RELEASE_COUNT)) { |
783 | ret = t7xx_dpmaifq_rx_notify_hw(rxq); |
784 | if (ret < 0) |
785 | break; |
786 | } |
787 | } |
788 | |
789 | if (!ret) |
790 | ret = t7xx_dpmaifq_rx_notify_hw(rxq); |
791 | |
792 | if (ret) |
793 | return ret; |
794 | |
795 | return recv_skb_cnt; |
796 | } |
797 | |
798 | static unsigned int t7xx_dpmaifq_poll_pit(struct dpmaif_rx_queue *rxq) |
799 | { |
800 | unsigned int hw_wr_idx, pit_cnt; |
801 | |
802 | if (!rxq->que_started) |
803 | return 0; |
804 | |
805 | hw_wr_idx = t7xx_dpmaif_dl_dlq_pit_get_wr_idx(hw_info: &rxq->dpmaif_ctrl->hw_info, dlq_pit_idx: rxq->index); |
806 | pit_cnt = t7xx_ring_buf_rd_wr_count(total_cnt: rxq->pit_size_cnt, rd_idx: rxq->pit_rd_idx, wr_idx: hw_wr_idx, |
807 | DPMAIF_READ); |
808 | rxq->pit_wr_idx = hw_wr_idx; |
809 | return pit_cnt; |
810 | } |
811 | |
812 | static int t7xx_dpmaif_napi_rx_data_collect(struct dpmaif_ctrl *dpmaif_ctrl, |
813 | const unsigned int q_num, |
814 | const unsigned int budget, int *once_more) |
815 | { |
816 | struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[q_num]; |
817 | unsigned int cnt; |
818 | int ret = 0; |
819 | |
820 | cnt = t7xx_dpmaifq_poll_pit(rxq); |
821 | if (!cnt) |
822 | return ret; |
823 | |
824 | ret = t7xx_dpmaif_rx_start(rxq, pit_cnt: cnt, budget, once_more); |
825 | if (ret < 0) |
826 | dev_err(dpmaif_ctrl->dev, "dlq%u rx ERR:%d\n" , rxq->index, ret); |
827 | |
828 | return ret; |
829 | } |
830 | |
831 | int t7xx_dpmaif_napi_rx_poll(struct napi_struct *napi, const int budget) |
832 | { |
833 | struct dpmaif_rx_queue *rxq = container_of(napi, struct dpmaif_rx_queue, napi); |
834 | struct t7xx_pci_dev *t7xx_dev = rxq->dpmaif_ctrl->t7xx_dev; |
835 | int ret, once_more = 0, work_done = 0; |
836 | |
837 | atomic_set(v: &rxq->rx_processing, i: 1); |
838 | /* Ensure rx_processing is changed to 1 before actually begin RX flow */ |
839 | smp_mb(); |
840 | |
841 | if (!rxq->que_started) { |
842 | atomic_set(v: &rxq->rx_processing, i: 0); |
843 | pm_runtime_put_autosuspend(dev: rxq->dpmaif_ctrl->dev); |
844 | dev_err(rxq->dpmaif_ctrl->dev, "Work RXQ: %d has not been started\n" , rxq->index); |
845 | return work_done; |
846 | } |
847 | |
848 | if (!rxq->sleep_lock_pending) |
849 | t7xx_pci_disable_sleep(t7xx_dev); |
850 | |
851 | ret = try_wait_for_completion(x: &t7xx_dev->sleep_lock_acquire); |
852 | if (!ret) { |
853 | napi_complete_done(n: napi, work_done); |
854 | rxq->sleep_lock_pending = true; |
855 | napi_schedule(n: napi); |
856 | return work_done; |
857 | } |
858 | |
859 | rxq->sleep_lock_pending = false; |
860 | while (work_done < budget) { |
861 | int each_budget = budget - work_done; |
862 | int rx_cnt = t7xx_dpmaif_napi_rx_data_collect(dpmaif_ctrl: rxq->dpmaif_ctrl, q_num: rxq->index, |
863 | budget: each_budget, once_more: &once_more); |
864 | if (rx_cnt > 0) |
865 | work_done += rx_cnt; |
866 | else |
867 | break; |
868 | } |
869 | |
870 | if (once_more) { |
871 | napi_gro_flush(napi, flush_old: false); |
872 | work_done = budget; |
873 | t7xx_dpmaif_clr_ip_busy_sts(hw_info: &rxq->dpmaif_ctrl->hw_info); |
874 | } else if (work_done < budget) { |
875 | napi_complete_done(n: napi, work_done); |
876 | t7xx_dpmaif_clr_ip_busy_sts(hw_info: &rxq->dpmaif_ctrl->hw_info); |
877 | t7xx_dpmaif_dlq_unmask_rx_done(hw_info: &rxq->dpmaif_ctrl->hw_info, qno: rxq->index); |
878 | t7xx_pci_enable_sleep(t7xx_dev: rxq->dpmaif_ctrl->t7xx_dev); |
879 | pm_runtime_mark_last_busy(dev: rxq->dpmaif_ctrl->dev); |
880 | pm_runtime_put_autosuspend(dev: rxq->dpmaif_ctrl->dev); |
881 | atomic_set(v: &rxq->rx_processing, i: 0); |
882 | } else { |
883 | t7xx_dpmaif_clr_ip_busy_sts(hw_info: &rxq->dpmaif_ctrl->hw_info); |
884 | } |
885 | |
886 | return work_done; |
887 | } |
888 | |
889 | void t7xx_dpmaif_irq_rx_done(struct dpmaif_ctrl *dpmaif_ctrl, const unsigned int que_mask) |
890 | { |
891 | struct dpmaif_rx_queue *rxq; |
892 | struct dpmaif_ctrl *ctrl; |
893 | int qno, ret; |
894 | |
895 | qno = ffs(que_mask) - 1; |
896 | if (qno < 0 || qno > DPMAIF_RXQ_NUM - 1) { |
897 | dev_err(dpmaif_ctrl->dev, "Invalid RXQ number: %u\n" , qno); |
898 | return; |
899 | } |
900 | |
901 | rxq = &dpmaif_ctrl->rxq[qno]; |
902 | ctrl = rxq->dpmaif_ctrl; |
903 | /* We need to make sure that the modem has been resumed before |
904 | * calling napi. This can't be done inside the polling function |
905 | * as we could be blocked waiting for device to be resumed, |
906 | * which can't be done from softirq context the poll function |
907 | * is running in. |
908 | */ |
909 | ret = pm_runtime_resume_and_get(dev: ctrl->dev); |
910 | if (ret < 0 && ret != -EACCES) { |
911 | dev_err(ctrl->dev, "Failed to resume device: %d\n" , ret); |
912 | return; |
913 | } |
914 | napi_schedule(n: &rxq->napi); |
915 | } |
916 | |
917 | static void t7xx_dpmaif_base_free(const struct dpmaif_ctrl *dpmaif_ctrl, |
918 | const struct dpmaif_bat_request *bat_req) |
919 | { |
920 | if (bat_req->bat_base) |
921 | dma_free_coherent(dev: dpmaif_ctrl->dev, |
922 | size: bat_req->bat_size_cnt * sizeof(struct dpmaif_bat), |
923 | cpu_addr: bat_req->bat_base, dma_handle: bat_req->bat_bus_addr); |
924 | } |
925 | |
926 | /** |
927 | * t7xx_dpmaif_bat_alloc() - Allocate the BAT ring buffer. |
928 | * @dpmaif_ctrl: Pointer to DPMAIF context structure. |
929 | * @bat_req: Pointer to BAT request structure. |
930 | * @buf_type: BAT ring type. |
931 | * |
932 | * This function allocates the BAT ring buffer shared with the HW device, also allocates |
933 | * a buffer used to store information about the BAT skbs for further release. |
934 | * |
935 | * Return: |
936 | * * 0 - Success. |
937 | * * -ERROR - Error code. |
938 | */ |
939 | int t7xx_dpmaif_bat_alloc(const struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req, |
940 | const enum bat_type buf_type) |
941 | { |
942 | int sw_buf_size; |
943 | |
944 | if (buf_type == BAT_TYPE_FRAG) { |
945 | sw_buf_size = sizeof(struct dpmaif_bat_page); |
946 | bat_req->bat_size_cnt = DPMAIF_FRG_COUNT; |
947 | bat_req->pkt_buf_sz = DPMAIF_HW_FRG_PKTBUF; |
948 | } else { |
949 | sw_buf_size = sizeof(struct dpmaif_bat_skb); |
950 | bat_req->bat_size_cnt = DPMAIF_BAT_COUNT; |
951 | bat_req->pkt_buf_sz = DPMAIF_HW_BAT_PKTBUF; |
952 | } |
953 | |
954 | bat_req->type = buf_type; |
955 | bat_req->bat_wr_idx = 0; |
956 | bat_req->bat_release_rd_idx = 0; |
957 | |
958 | bat_req->bat_base = dma_alloc_coherent(dev: dpmaif_ctrl->dev, |
959 | size: bat_req->bat_size_cnt * sizeof(struct dpmaif_bat), |
960 | dma_handle: &bat_req->bat_bus_addr, GFP_KERNEL | __GFP_ZERO); |
961 | if (!bat_req->bat_base) |
962 | return -ENOMEM; |
963 | |
964 | /* For AP SW to record skb information */ |
965 | bat_req->bat_skb = devm_kzalloc(dev: dpmaif_ctrl->dev, size: bat_req->bat_size_cnt * sw_buf_size, |
966 | GFP_KERNEL); |
967 | if (!bat_req->bat_skb) |
968 | goto err_free_dma_mem; |
969 | |
970 | bat_req->bat_bitmap = bitmap_zalloc(nbits: bat_req->bat_size_cnt, GFP_KERNEL); |
971 | if (!bat_req->bat_bitmap) |
972 | goto err_free_dma_mem; |
973 | |
974 | spin_lock_init(&bat_req->mask_lock); |
975 | atomic_set(v: &bat_req->refcnt, i: 0); |
976 | return 0; |
977 | |
978 | err_free_dma_mem: |
979 | t7xx_dpmaif_base_free(dpmaif_ctrl, bat_req); |
980 | |
981 | return -ENOMEM; |
982 | } |
983 | |
984 | void t7xx_dpmaif_bat_free(const struct dpmaif_ctrl *dpmaif_ctrl, struct dpmaif_bat_request *bat_req) |
985 | { |
986 | if (!bat_req || !atomic_dec_and_test(v: &bat_req->refcnt)) |
987 | return; |
988 | |
989 | bitmap_free(bitmap: bat_req->bat_bitmap); |
990 | bat_req->bat_bitmap = NULL; |
991 | |
992 | if (bat_req->bat_skb) { |
993 | unsigned int i; |
994 | |
995 | for (i = 0; i < bat_req->bat_size_cnt; i++) { |
996 | if (bat_req->type == BAT_TYPE_FRAG) |
997 | t7xx_unmap_bat_page(dev: dpmaif_ctrl->dev, bat_page_base: bat_req->bat_skb, index: i); |
998 | else |
999 | t7xx_unmap_bat_skb(dev: dpmaif_ctrl->dev, bat_skb_base: bat_req->bat_skb, index: i); |
1000 | } |
1001 | } |
1002 | |
1003 | t7xx_dpmaif_base_free(dpmaif_ctrl, bat_req); |
1004 | } |
1005 | |
1006 | static int t7xx_dpmaif_rx_alloc(struct dpmaif_rx_queue *rxq) |
1007 | { |
1008 | rxq->pit_size_cnt = DPMAIF_PIT_COUNT; |
1009 | rxq->pit_rd_idx = 0; |
1010 | rxq->pit_wr_idx = 0; |
1011 | rxq->pit_release_rd_idx = 0; |
1012 | rxq->expect_pit_seq = 0; |
1013 | rxq->pit_remain_release_cnt = 0; |
1014 | memset(&rxq->rx_data_info, 0, sizeof(rxq->rx_data_info)); |
1015 | |
1016 | rxq->pit_base = dma_alloc_coherent(dev: rxq->dpmaif_ctrl->dev, |
1017 | size: rxq->pit_size_cnt * sizeof(struct dpmaif_pit), |
1018 | dma_handle: &rxq->pit_bus_addr, GFP_KERNEL | __GFP_ZERO); |
1019 | if (!rxq->pit_base) |
1020 | return -ENOMEM; |
1021 | |
1022 | rxq->bat_req = &rxq->dpmaif_ctrl->bat_req; |
1023 | atomic_inc(v: &rxq->bat_req->refcnt); |
1024 | |
1025 | rxq->bat_frag = &rxq->dpmaif_ctrl->bat_frag; |
1026 | atomic_inc(v: &rxq->bat_frag->refcnt); |
1027 | return 0; |
1028 | } |
1029 | |
1030 | static void t7xx_dpmaif_rx_buf_free(const struct dpmaif_rx_queue *rxq) |
1031 | { |
1032 | if (!rxq->dpmaif_ctrl) |
1033 | return; |
1034 | |
1035 | t7xx_dpmaif_bat_free(dpmaif_ctrl: rxq->dpmaif_ctrl, bat_req: rxq->bat_req); |
1036 | t7xx_dpmaif_bat_free(dpmaif_ctrl: rxq->dpmaif_ctrl, bat_req: rxq->bat_frag); |
1037 | |
1038 | if (rxq->pit_base) |
1039 | dma_free_coherent(dev: rxq->dpmaif_ctrl->dev, |
1040 | size: rxq->pit_size_cnt * sizeof(struct dpmaif_pit), |
1041 | cpu_addr: rxq->pit_base, dma_handle: rxq->pit_bus_addr); |
1042 | } |
1043 | |
1044 | int t7xx_dpmaif_rxq_init(struct dpmaif_rx_queue *queue) |
1045 | { |
1046 | int ret; |
1047 | |
1048 | ret = t7xx_dpmaif_rx_alloc(rxq: queue); |
1049 | if (ret < 0) |
1050 | dev_err(queue->dpmaif_ctrl->dev, "Failed to allocate RX buffers: %d\n" , ret); |
1051 | |
1052 | return ret; |
1053 | } |
1054 | |
1055 | void t7xx_dpmaif_rxq_free(struct dpmaif_rx_queue *queue) |
1056 | { |
1057 | t7xx_dpmaif_rx_buf_free(rxq: queue); |
1058 | } |
1059 | |
1060 | static void t7xx_dpmaif_bat_release_work(struct work_struct *work) |
1061 | { |
1062 | struct dpmaif_ctrl *dpmaif_ctrl = container_of(work, struct dpmaif_ctrl, bat_release_work); |
1063 | struct dpmaif_rx_queue *rxq; |
1064 | int ret; |
1065 | |
1066 | ret = pm_runtime_resume_and_get(dev: dpmaif_ctrl->dev); |
1067 | if (ret < 0 && ret != -EACCES) |
1068 | return; |
1069 | |
1070 | t7xx_pci_disable_sleep(t7xx_dev: dpmaif_ctrl->t7xx_dev); |
1071 | |
1072 | /* ALL RXQ use one BAT table, so choose DPF_RX_QNO_DFT */ |
1073 | rxq = &dpmaif_ctrl->rxq[DPF_RX_QNO_DFT]; |
1074 | if (t7xx_pci_sleep_disable_complete(t7xx_dev: dpmaif_ctrl->t7xx_dev)) { |
1075 | t7xx_dpmaif_bat_release_and_add(rxq); |
1076 | t7xx_dpmaif_frag_bat_release_and_add(rxq); |
1077 | } |
1078 | |
1079 | t7xx_pci_enable_sleep(t7xx_dev: dpmaif_ctrl->t7xx_dev); |
1080 | pm_runtime_mark_last_busy(dev: dpmaif_ctrl->dev); |
1081 | pm_runtime_put_autosuspend(dev: dpmaif_ctrl->dev); |
1082 | } |
1083 | |
1084 | int t7xx_dpmaif_bat_rel_wq_alloc(struct dpmaif_ctrl *dpmaif_ctrl) |
1085 | { |
1086 | dpmaif_ctrl->bat_release_wq = alloc_workqueue(fmt: "dpmaif_bat_release_work_queue" , |
1087 | flags: WQ_MEM_RECLAIM, max_active: 1); |
1088 | if (!dpmaif_ctrl->bat_release_wq) |
1089 | return -ENOMEM; |
1090 | |
1091 | INIT_WORK(&dpmaif_ctrl->bat_release_work, t7xx_dpmaif_bat_release_work); |
1092 | return 0; |
1093 | } |
1094 | |
1095 | void t7xx_dpmaif_bat_wq_rel(struct dpmaif_ctrl *dpmaif_ctrl) |
1096 | { |
1097 | flush_work(work: &dpmaif_ctrl->bat_release_work); |
1098 | |
1099 | if (dpmaif_ctrl->bat_release_wq) { |
1100 | destroy_workqueue(wq: dpmaif_ctrl->bat_release_wq); |
1101 | dpmaif_ctrl->bat_release_wq = NULL; |
1102 | } |
1103 | } |
1104 | |
1105 | /** |
1106 | * t7xx_dpmaif_rx_stop() - Suspend RX flow. |
1107 | * @dpmaif_ctrl: Pointer to data path control struct dpmaif_ctrl. |
1108 | * |
1109 | * Wait for all the RX work to finish executing and mark the RX queue as paused. |
1110 | */ |
1111 | void t7xx_dpmaif_rx_stop(struct dpmaif_ctrl *dpmaif_ctrl) |
1112 | { |
1113 | unsigned int i; |
1114 | |
1115 | for (i = 0; i < DPMAIF_RXQ_NUM; i++) { |
1116 | struct dpmaif_rx_queue *rxq = &dpmaif_ctrl->rxq[i]; |
1117 | int timeout, value; |
1118 | |
1119 | timeout = readx_poll_timeout_atomic(atomic_read, &rxq->rx_processing, value, |
1120 | !value, 0, DPMAIF_CHECK_INIT_TIMEOUT_US); |
1121 | if (timeout) |
1122 | dev_err(dpmaif_ctrl->dev, "Stop RX SW failed\n" ); |
1123 | |
1124 | /* Ensure RX processing has stopped before we set rxq->que_started to false */ |
1125 | smp_mb(); |
1126 | rxq->que_started = false; |
1127 | } |
1128 | } |
1129 | |
1130 | static void t7xx_dpmaif_stop_rxq(struct dpmaif_rx_queue *rxq) |
1131 | { |
1132 | int cnt, j = 0; |
1133 | |
1134 | rxq->que_started = false; |
1135 | |
1136 | do { |
1137 | cnt = t7xx_ring_buf_rd_wr_count(total_cnt: rxq->pit_size_cnt, rd_idx: rxq->pit_rd_idx, |
1138 | wr_idx: rxq->pit_wr_idx, DPMAIF_READ); |
1139 | |
1140 | if (++j >= DPMAIF_MAX_CHECK_COUNT) { |
1141 | dev_err(rxq->dpmaif_ctrl->dev, "Stop RX SW failed, %d\n" , cnt); |
1142 | break; |
1143 | } |
1144 | } while (cnt); |
1145 | |
1146 | memset(rxq->pit_base, 0, rxq->pit_size_cnt * sizeof(struct dpmaif_pit)); |
1147 | memset(rxq->bat_req->bat_base, 0, rxq->bat_req->bat_size_cnt * sizeof(struct dpmaif_bat)); |
1148 | bitmap_zero(dst: rxq->bat_req->bat_bitmap, nbits: rxq->bat_req->bat_size_cnt); |
1149 | memset(&rxq->rx_data_info, 0, sizeof(rxq->rx_data_info)); |
1150 | |
1151 | rxq->pit_rd_idx = 0; |
1152 | rxq->pit_wr_idx = 0; |
1153 | rxq->pit_release_rd_idx = 0; |
1154 | rxq->expect_pit_seq = 0; |
1155 | rxq->pit_remain_release_cnt = 0; |
1156 | rxq->bat_req->bat_release_rd_idx = 0; |
1157 | rxq->bat_req->bat_wr_idx = 0; |
1158 | rxq->bat_frag->bat_release_rd_idx = 0; |
1159 | rxq->bat_frag->bat_wr_idx = 0; |
1160 | } |
1161 | |
1162 | void t7xx_dpmaif_rx_clear(struct dpmaif_ctrl *dpmaif_ctrl) |
1163 | { |
1164 | int i; |
1165 | |
1166 | for (i = 0; i < DPMAIF_RXQ_NUM; i++) |
1167 | t7xx_dpmaif_stop_rxq(rxq: &dpmaif_ctrl->rxq[i]); |
1168 | } |
1169 | |