1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Texas Instruments CPDMA Driver |
4 | * |
5 | * Copyright (C) 2010 Texas Instruments |
6 | * |
7 | */ |
8 | #include <linux/kernel.h> |
9 | #include <linux/spinlock.h> |
10 | #include <linux/device.h> |
11 | #include <linux/module.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/err.h> |
14 | #include <linux/dma-mapping.h> |
15 | #include <linux/io.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/genalloc.h> |
18 | #include "davinci_cpdma.h" |
19 | |
20 | /* DMA Registers */ |
21 | #define CPDMA_TXIDVER 0x00 |
22 | #define CPDMA_TXCONTROL 0x04 |
23 | #define CPDMA_TXTEARDOWN 0x08 |
24 | #define CPDMA_RXIDVER 0x10 |
25 | #define CPDMA_RXCONTROL 0x14 |
26 | #define CPDMA_SOFTRESET 0x1c |
27 | #define CPDMA_RXTEARDOWN 0x18 |
28 | #define CPDMA_TX_PRI0_RATE 0x30 |
29 | #define CPDMA_TXINTSTATRAW 0x80 |
30 | #define CPDMA_TXINTSTATMASKED 0x84 |
31 | #define CPDMA_TXINTMASKSET 0x88 |
32 | #define CPDMA_TXINTMASKCLEAR 0x8c |
33 | #define CPDMA_MACINVECTOR 0x90 |
34 | #define CPDMA_MACEOIVECTOR 0x94 |
35 | #define CPDMA_RXINTSTATRAW 0xa0 |
36 | #define CPDMA_RXINTSTATMASKED 0xa4 |
37 | #define CPDMA_RXINTMASKSET 0xa8 |
38 | #define CPDMA_RXINTMASKCLEAR 0xac |
39 | #define CPDMA_DMAINTSTATRAW 0xb0 |
40 | #define CPDMA_DMAINTSTATMASKED 0xb4 |
41 | #define CPDMA_DMAINTMASKSET 0xb8 |
42 | #define CPDMA_DMAINTMASKCLEAR 0xbc |
43 | #define CPDMA_DMAINT_HOSTERR BIT(1) |
44 | |
45 | /* the following exist only if has_ext_regs is set */ |
46 | #define CPDMA_DMACONTROL 0x20 |
47 | #define CPDMA_DMASTATUS 0x24 |
48 | #define CPDMA_RXBUFFOFS 0x28 |
49 | #define CPDMA_EM_CONTROL 0x2c |
50 | |
51 | /* Descriptor mode bits */ |
52 | #define CPDMA_DESC_SOP BIT(31) |
53 | #define CPDMA_DESC_EOP BIT(30) |
54 | #define CPDMA_DESC_OWNER BIT(29) |
55 | #define CPDMA_DESC_EOQ BIT(28) |
56 | #define CPDMA_DESC_TD_COMPLETE BIT(27) |
57 | #define CPDMA_DESC_PASS_CRC BIT(26) |
58 | #define CPDMA_DESC_TO_PORT_EN BIT(20) |
59 | #define CPDMA_TO_PORT_SHIFT 16 |
60 | #define CPDMA_DESC_PORT_MASK (BIT(18) | BIT(17) | BIT(16)) |
61 | #define CPDMA_DESC_CRC_LEN 4 |
62 | |
63 | #define CPDMA_TEARDOWN_VALUE 0xfffffffc |
64 | |
65 | #define CPDMA_MAX_RLIM_CNT 16384 |
66 | |
67 | struct cpdma_desc { |
68 | /* hardware fields */ |
69 | u32 hw_next; |
70 | u32 hw_buffer; |
71 | u32 hw_len; |
72 | u32 hw_mode; |
73 | /* software fields */ |
74 | void *sw_token; |
75 | u32 sw_buffer; |
76 | u32 sw_len; |
77 | }; |
78 | |
79 | struct cpdma_desc_pool { |
80 | phys_addr_t phys; |
81 | dma_addr_t hw_addr; |
82 | void __iomem *iomap; /* ioremap map */ |
83 | void *cpumap; /* dma_alloc map */ |
84 | int desc_size, mem_size; |
85 | int num_desc; |
86 | struct device *dev; |
87 | struct gen_pool *gen_pool; |
88 | }; |
89 | |
90 | enum cpdma_state { |
91 | CPDMA_STATE_IDLE, |
92 | CPDMA_STATE_ACTIVE, |
93 | CPDMA_STATE_TEARDOWN, |
94 | }; |
95 | |
96 | struct cpdma_ctlr { |
97 | enum cpdma_state state; |
98 | struct cpdma_params params; |
99 | struct device *dev; |
100 | struct cpdma_desc_pool *pool; |
101 | spinlock_t lock; |
102 | struct cpdma_chan *channels[2 * CPDMA_MAX_CHANNELS]; |
103 | int chan_num; |
104 | int num_rx_desc; /* RX descriptors number */ |
105 | int num_tx_desc; /* TX descriptors number */ |
106 | }; |
107 | |
108 | struct cpdma_chan { |
109 | struct cpdma_desc __iomem *head, *tail; |
110 | void __iomem *hdp, *cp, *rxfree; |
111 | enum cpdma_state state; |
112 | struct cpdma_ctlr *ctlr; |
113 | int chan_num; |
114 | spinlock_t lock; |
115 | int count; |
116 | u32 desc_num; |
117 | u32 mask; |
118 | cpdma_handler_fn handler; |
119 | enum dma_data_direction dir; |
120 | struct cpdma_chan_stats stats; |
121 | /* offsets into dmaregs */ |
122 | int int_set, int_clear, td; |
123 | int weight; |
124 | u32 rate_factor; |
125 | u32 rate; |
126 | }; |
127 | |
128 | struct cpdma_control_info { |
129 | u32 reg; |
130 | u32 shift, mask; |
131 | int access; |
132 | #define ACCESS_RO BIT(0) |
133 | #define ACCESS_WO BIT(1) |
134 | #define ACCESS_RW (ACCESS_RO | ACCESS_WO) |
135 | }; |
136 | |
137 | struct submit_info { |
138 | struct cpdma_chan *chan; |
139 | int directed; |
140 | void *token; |
141 | void *data_virt; |
142 | dma_addr_t data_dma; |
143 | int len; |
144 | }; |
145 | |
146 | static struct cpdma_control_info controls[] = { |
147 | [CPDMA_TX_RLIM] = {CPDMA_DMACONTROL, 8, 0xffff, ACCESS_RW}, |
148 | [CPDMA_CMD_IDLE] = {CPDMA_DMACONTROL, .shift: 3, .mask: 1, ACCESS_WO}, |
149 | [CPDMA_COPY_ERROR_FRAMES] = {CPDMA_DMACONTROL, .shift: 4, .mask: 1, ACCESS_RW}, |
150 | [CPDMA_RX_OFF_LEN_UPDATE] = {CPDMA_DMACONTROL, .shift: 2, .mask: 1, ACCESS_RW}, |
151 | [CPDMA_RX_OWNERSHIP_FLIP] = {CPDMA_DMACONTROL, .shift: 1, .mask: 1, ACCESS_RW}, |
152 | [CPDMA_TX_PRIO_FIXED] = {CPDMA_DMACONTROL, .shift: 0, .mask: 1, ACCESS_RW}, |
153 | [CPDMA_STAT_IDLE] = {CPDMA_DMASTATUS, .shift: 31, .mask: 1, ACCESS_RO}, |
154 | [CPDMA_STAT_TX_ERR_CODE] = {CPDMA_DMASTATUS, .shift: 20, .mask: 0xf, ACCESS_RW}, |
155 | [CPDMA_STAT_TX_ERR_CHAN] = {CPDMA_DMASTATUS, .shift: 16, .mask: 0x7, ACCESS_RW}, |
156 | [CPDMA_STAT_RX_ERR_CODE] = {CPDMA_DMASTATUS, .shift: 12, .mask: 0xf, ACCESS_RW}, |
157 | [CPDMA_STAT_RX_ERR_CHAN] = {CPDMA_DMASTATUS, .shift: 8, .mask: 0x7, ACCESS_RW}, |
158 | [CPDMA_RX_BUFFER_OFFSET] = {CPDMA_RXBUFFOFS, .shift: 0, .mask: 0xffff, ACCESS_RW}, |
159 | }; |
160 | |
161 | #define tx_chan_num(chan) (chan) |
162 | #define rx_chan_num(chan) ((chan) + CPDMA_MAX_CHANNELS) |
163 | #define is_rx_chan(chan) ((chan)->chan_num >= CPDMA_MAX_CHANNELS) |
164 | #define is_tx_chan(chan) (!is_rx_chan(chan)) |
165 | #define __chan_linear(chan_num) ((chan_num) & (CPDMA_MAX_CHANNELS - 1)) |
166 | #define chan_linear(chan) __chan_linear((chan)->chan_num) |
167 | |
168 | /* The following make access to common cpdma_ctlr params more readable */ |
169 | #define dmaregs params.dmaregs |
170 | #define num_chan params.num_chan |
171 | |
172 | /* various accessors */ |
173 | #define dma_reg_read(ctlr, ofs) readl((ctlr)->dmaregs + (ofs)) |
174 | #define chan_read(chan, fld) readl((chan)->fld) |
175 | #define desc_read(desc, fld) readl(&(desc)->fld) |
176 | #define dma_reg_write(ctlr, ofs, v) writel(v, (ctlr)->dmaregs + (ofs)) |
177 | #define chan_write(chan, fld, v) writel(v, (chan)->fld) |
178 | #define desc_write(desc, fld, v) writel((u32)(v), &(desc)->fld) |
179 | |
180 | #define cpdma_desc_to_port(chan, mode, directed) \ |
181 | do { \ |
182 | if (!is_rx_chan(chan) && ((directed == 1) || \ |
183 | (directed == 2))) \ |
184 | mode |= (CPDMA_DESC_TO_PORT_EN | \ |
185 | (directed << CPDMA_TO_PORT_SHIFT)); \ |
186 | } while (0) |
187 | |
188 | #define CPDMA_DMA_EXT_MAP BIT(16) |
189 | |
190 | static void cpdma_desc_pool_destroy(struct cpdma_ctlr *ctlr) |
191 | { |
192 | struct cpdma_desc_pool *pool = ctlr->pool; |
193 | |
194 | if (!pool) |
195 | return; |
196 | |
197 | WARN(gen_pool_size(pool->gen_pool) != gen_pool_avail(pool->gen_pool), |
198 | "cpdma_desc_pool size %zd != avail %zd" , |
199 | gen_pool_size(pool->gen_pool), |
200 | gen_pool_avail(pool->gen_pool)); |
201 | if (pool->cpumap) |
202 | dma_free_coherent(dev: ctlr->dev, size: pool->mem_size, cpu_addr: pool->cpumap, |
203 | dma_handle: pool->phys); |
204 | } |
205 | |
206 | /* |
207 | * Utility constructs for a cpdma descriptor pool. Some devices (e.g. davinci |
208 | * emac) have dedicated on-chip memory for these descriptors. Some other |
209 | * devices (e.g. cpsw switches) use plain old memory. Descriptor pools |
210 | * abstract out these details |
211 | */ |
212 | static int cpdma_desc_pool_create(struct cpdma_ctlr *ctlr) |
213 | { |
214 | struct cpdma_params *cpdma_params = &ctlr->params; |
215 | struct cpdma_desc_pool *pool; |
216 | int ret = -ENOMEM; |
217 | |
218 | pool = devm_kzalloc(dev: ctlr->dev, size: sizeof(*pool), GFP_KERNEL); |
219 | if (!pool) |
220 | goto gen_pool_create_fail; |
221 | ctlr->pool = pool; |
222 | |
223 | pool->mem_size = cpdma_params->desc_mem_size; |
224 | pool->desc_size = ALIGN(sizeof(struct cpdma_desc), |
225 | cpdma_params->desc_align); |
226 | pool->num_desc = pool->mem_size / pool->desc_size; |
227 | |
228 | if (cpdma_params->descs_pool_size) { |
229 | /* recalculate memory size required cpdma descriptor pool |
230 | * basing on number of descriptors specified by user and |
231 | * if memory size > CPPI internal RAM size (desc_mem_size) |
232 | * then switch to use DDR |
233 | */ |
234 | pool->num_desc = cpdma_params->descs_pool_size; |
235 | pool->mem_size = pool->desc_size * pool->num_desc; |
236 | if (pool->mem_size > cpdma_params->desc_mem_size) |
237 | cpdma_params->desc_mem_phys = 0; |
238 | } |
239 | |
240 | pool->gen_pool = devm_gen_pool_create(dev: ctlr->dev, ilog2(pool->desc_size), |
241 | nid: -1, name: "cpdma" ); |
242 | if (IS_ERR(ptr: pool->gen_pool)) { |
243 | ret = PTR_ERR(ptr: pool->gen_pool); |
244 | dev_err(ctlr->dev, "pool create failed %d\n" , ret); |
245 | goto gen_pool_create_fail; |
246 | } |
247 | |
248 | if (cpdma_params->desc_mem_phys) { |
249 | pool->phys = cpdma_params->desc_mem_phys; |
250 | pool->iomap = devm_ioremap(dev: ctlr->dev, offset: pool->phys, |
251 | size: pool->mem_size); |
252 | pool->hw_addr = cpdma_params->desc_hw_addr; |
253 | } else { |
254 | pool->cpumap = dma_alloc_coherent(dev: ctlr->dev, size: pool->mem_size, |
255 | dma_handle: &pool->hw_addr, GFP_KERNEL); |
256 | pool->iomap = (void __iomem __force *)pool->cpumap; |
257 | pool->phys = pool->hw_addr; /* assumes no IOMMU, don't use this value */ |
258 | } |
259 | |
260 | if (!pool->iomap) |
261 | goto gen_pool_create_fail; |
262 | |
263 | ret = gen_pool_add_virt(pool: pool->gen_pool, addr: (unsigned long)pool->iomap, |
264 | phys: pool->phys, size: pool->mem_size, nid: -1); |
265 | if (ret < 0) { |
266 | dev_err(ctlr->dev, "pool add failed %d\n" , ret); |
267 | goto gen_pool_add_virt_fail; |
268 | } |
269 | |
270 | return 0; |
271 | |
272 | gen_pool_add_virt_fail: |
273 | cpdma_desc_pool_destroy(ctlr); |
274 | gen_pool_create_fail: |
275 | ctlr->pool = NULL; |
276 | return ret; |
277 | } |
278 | |
279 | static inline dma_addr_t desc_phys(struct cpdma_desc_pool *pool, |
280 | struct cpdma_desc __iomem *desc) |
281 | { |
282 | if (!desc) |
283 | return 0; |
284 | return pool->hw_addr + (__force long)desc - (__force long)pool->iomap; |
285 | } |
286 | |
287 | static inline struct cpdma_desc __iomem * |
288 | desc_from_phys(struct cpdma_desc_pool *pool, dma_addr_t dma) |
289 | { |
290 | return dma ? pool->iomap + dma - pool->hw_addr : NULL; |
291 | } |
292 | |
293 | static struct cpdma_desc __iomem * |
294 | cpdma_desc_alloc(struct cpdma_desc_pool *pool) |
295 | { |
296 | return (struct cpdma_desc __iomem *) |
297 | gen_pool_alloc(pool: pool->gen_pool, size: pool->desc_size); |
298 | } |
299 | |
300 | static void cpdma_desc_free(struct cpdma_desc_pool *pool, |
301 | struct cpdma_desc __iomem *desc, int num_desc) |
302 | { |
303 | gen_pool_free(pool: pool->gen_pool, addr: (unsigned long)desc, size: pool->desc_size); |
304 | } |
305 | |
306 | static int _cpdma_control_set(struct cpdma_ctlr *ctlr, int control, int value) |
307 | { |
308 | struct cpdma_control_info *info = &controls[control]; |
309 | u32 val; |
310 | |
311 | if (!ctlr->params.has_ext_regs) |
312 | return -ENOTSUPP; |
313 | |
314 | if (ctlr->state != CPDMA_STATE_ACTIVE) |
315 | return -EINVAL; |
316 | |
317 | if (control < 0 || control >= ARRAY_SIZE(controls)) |
318 | return -ENOENT; |
319 | |
320 | if ((info->access & ACCESS_WO) != ACCESS_WO) |
321 | return -EPERM; |
322 | |
323 | val = dma_reg_read(ctlr, info->reg); |
324 | val &= ~(info->mask << info->shift); |
325 | val |= (value & info->mask) << info->shift; |
326 | dma_reg_write(ctlr, info->reg, val); |
327 | |
328 | return 0; |
329 | } |
330 | |
331 | static int _cpdma_control_get(struct cpdma_ctlr *ctlr, int control) |
332 | { |
333 | struct cpdma_control_info *info = &controls[control]; |
334 | int ret; |
335 | |
336 | if (!ctlr->params.has_ext_regs) |
337 | return -ENOTSUPP; |
338 | |
339 | if (ctlr->state != CPDMA_STATE_ACTIVE) |
340 | return -EINVAL; |
341 | |
342 | if (control < 0 || control >= ARRAY_SIZE(controls)) |
343 | return -ENOENT; |
344 | |
345 | if ((info->access & ACCESS_RO) != ACCESS_RO) |
346 | return -EPERM; |
347 | |
348 | ret = (dma_reg_read(ctlr, info->reg) >> info->shift) & info->mask; |
349 | return ret; |
350 | } |
351 | |
352 | /* cpdma_chan_set_chan_shaper - set shaper for a channel |
353 | * Has to be called under ctlr lock |
354 | */ |
355 | static int cpdma_chan_set_chan_shaper(struct cpdma_chan *chan) |
356 | { |
357 | struct cpdma_ctlr *ctlr = chan->ctlr; |
358 | u32 rate_reg; |
359 | u32 rmask; |
360 | int ret; |
361 | |
362 | if (!chan->rate) |
363 | return 0; |
364 | |
365 | rate_reg = CPDMA_TX_PRI0_RATE + 4 * chan->chan_num; |
366 | dma_reg_write(ctlr, rate_reg, chan->rate_factor); |
367 | |
368 | rmask = _cpdma_control_get(ctlr, control: CPDMA_TX_RLIM); |
369 | rmask |= chan->mask; |
370 | |
371 | ret = _cpdma_control_set(ctlr, control: CPDMA_TX_RLIM, value: rmask); |
372 | return ret; |
373 | } |
374 | |
375 | static int cpdma_chan_on(struct cpdma_chan *chan) |
376 | { |
377 | struct cpdma_ctlr *ctlr = chan->ctlr; |
378 | struct cpdma_desc_pool *pool = ctlr->pool; |
379 | unsigned long flags; |
380 | |
381 | spin_lock_irqsave(&chan->lock, flags); |
382 | if (chan->state != CPDMA_STATE_IDLE) { |
383 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
384 | return -EBUSY; |
385 | } |
386 | if (ctlr->state != CPDMA_STATE_ACTIVE) { |
387 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
388 | return -EINVAL; |
389 | } |
390 | dma_reg_write(ctlr, chan->int_set, chan->mask); |
391 | chan->state = CPDMA_STATE_ACTIVE; |
392 | if (chan->head) { |
393 | chan_write(chan, hdp, desc_phys(pool, chan->head)); |
394 | if (chan->rxfree) |
395 | chan_write(chan, rxfree, chan->count); |
396 | } |
397 | |
398 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
399 | return 0; |
400 | } |
401 | |
402 | /* cpdma_chan_fit_rate - set rate for a channel and check if it's possible. |
403 | * rmask - mask of rate limited channels |
404 | * Returns min rate in Kb/s |
405 | */ |
406 | static int cpdma_chan_fit_rate(struct cpdma_chan *ch, u32 rate, |
407 | u32 *rmask, int *prio_mode) |
408 | { |
409 | struct cpdma_ctlr *ctlr = ch->ctlr; |
410 | struct cpdma_chan *chan; |
411 | u32 old_rate = ch->rate; |
412 | u32 new_rmask = 0; |
413 | int rlim = 0; |
414 | int i; |
415 | |
416 | for (i = tx_chan_num(0); i < tx_chan_num(CPDMA_MAX_CHANNELS); i++) { |
417 | chan = ctlr->channels[i]; |
418 | if (!chan) |
419 | continue; |
420 | |
421 | if (chan == ch) |
422 | chan->rate = rate; |
423 | |
424 | if (chan->rate) { |
425 | rlim = 1; |
426 | new_rmask |= chan->mask; |
427 | continue; |
428 | } |
429 | |
430 | if (rlim) |
431 | goto err; |
432 | } |
433 | |
434 | *rmask = new_rmask; |
435 | *prio_mode = rlim; |
436 | return 0; |
437 | |
438 | err: |
439 | ch->rate = old_rate; |
440 | dev_err(ctlr->dev, "Upper cpdma ch%d is not rate limited\n" , |
441 | chan->chan_num); |
442 | return -EINVAL; |
443 | } |
444 | |
445 | static u32 cpdma_chan_set_factors(struct cpdma_ctlr *ctlr, |
446 | struct cpdma_chan *ch) |
447 | { |
448 | u32 delta = UINT_MAX, prev_delta = UINT_MAX, best_delta = UINT_MAX; |
449 | u32 best_send_cnt = 0, best_idle_cnt = 0; |
450 | u32 new_rate, best_rate = 0, rate_reg; |
451 | u64 send_cnt, idle_cnt; |
452 | u32 min_send_cnt, freq; |
453 | u64 divident, divisor; |
454 | |
455 | if (!ch->rate) { |
456 | ch->rate_factor = 0; |
457 | goto set_factor; |
458 | } |
459 | |
460 | freq = ctlr->params.bus_freq_mhz * 1000 * 32; |
461 | if (!freq) { |
462 | dev_err(ctlr->dev, "The bus frequency is not set\n" ); |
463 | return -EINVAL; |
464 | } |
465 | |
466 | min_send_cnt = freq - ch->rate; |
467 | send_cnt = DIV_ROUND_UP(min_send_cnt, ch->rate); |
468 | while (send_cnt <= CPDMA_MAX_RLIM_CNT) { |
469 | divident = ch->rate * send_cnt; |
470 | divisor = min_send_cnt; |
471 | idle_cnt = DIV_ROUND_CLOSEST_ULL(divident, divisor); |
472 | |
473 | divident = freq * idle_cnt; |
474 | divisor = idle_cnt + send_cnt; |
475 | new_rate = DIV_ROUND_CLOSEST_ULL(divident, divisor); |
476 | |
477 | delta = new_rate >= ch->rate ? new_rate - ch->rate : delta; |
478 | if (delta < best_delta) { |
479 | best_delta = delta; |
480 | best_send_cnt = send_cnt; |
481 | best_idle_cnt = idle_cnt; |
482 | best_rate = new_rate; |
483 | |
484 | if (!delta) |
485 | break; |
486 | } |
487 | |
488 | if (prev_delta >= delta) { |
489 | prev_delta = delta; |
490 | send_cnt++; |
491 | continue; |
492 | } |
493 | |
494 | idle_cnt++; |
495 | divident = freq * idle_cnt; |
496 | send_cnt = DIV_ROUND_CLOSEST_ULL(divident, ch->rate); |
497 | send_cnt -= idle_cnt; |
498 | prev_delta = UINT_MAX; |
499 | } |
500 | |
501 | ch->rate = best_rate; |
502 | ch->rate_factor = best_send_cnt | (best_idle_cnt << 16); |
503 | |
504 | set_factor: |
505 | rate_reg = CPDMA_TX_PRI0_RATE + 4 * ch->chan_num; |
506 | dma_reg_write(ctlr, rate_reg, ch->rate_factor); |
507 | return 0; |
508 | } |
509 | |
510 | struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params) |
511 | { |
512 | struct cpdma_ctlr *ctlr; |
513 | |
514 | ctlr = devm_kzalloc(dev: params->dev, size: sizeof(*ctlr), GFP_KERNEL); |
515 | if (!ctlr) |
516 | return NULL; |
517 | |
518 | ctlr->state = CPDMA_STATE_IDLE; |
519 | ctlr->params = *params; |
520 | ctlr->dev = params->dev; |
521 | ctlr->chan_num = 0; |
522 | spin_lock_init(&ctlr->lock); |
523 | |
524 | if (cpdma_desc_pool_create(ctlr)) |
525 | return NULL; |
526 | /* split pool equally between RX/TX by default */ |
527 | ctlr->num_tx_desc = ctlr->pool->num_desc / 2; |
528 | ctlr->num_rx_desc = ctlr->pool->num_desc - ctlr->num_tx_desc; |
529 | |
530 | if (WARN_ON(ctlr->num_chan > CPDMA_MAX_CHANNELS)) |
531 | ctlr->num_chan = CPDMA_MAX_CHANNELS; |
532 | return ctlr; |
533 | } |
534 | |
535 | int cpdma_ctlr_start(struct cpdma_ctlr *ctlr) |
536 | { |
537 | struct cpdma_chan *chan; |
538 | unsigned long flags; |
539 | int i, prio_mode; |
540 | |
541 | spin_lock_irqsave(&ctlr->lock, flags); |
542 | if (ctlr->state != CPDMA_STATE_IDLE) { |
543 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
544 | return -EBUSY; |
545 | } |
546 | |
547 | if (ctlr->params.has_soft_reset) { |
548 | unsigned timeout = 10 * 100; |
549 | |
550 | dma_reg_write(ctlr, CPDMA_SOFTRESET, 1); |
551 | while (timeout) { |
552 | if (dma_reg_read(ctlr, CPDMA_SOFTRESET) == 0) |
553 | break; |
554 | udelay(10); |
555 | timeout--; |
556 | } |
557 | WARN_ON(!timeout); |
558 | } |
559 | |
560 | for (i = 0; i < ctlr->num_chan; i++) { |
561 | writel(val: 0, addr: ctlr->params.txhdp + 4 * i); |
562 | writel(val: 0, addr: ctlr->params.rxhdp + 4 * i); |
563 | writel(val: 0, addr: ctlr->params.txcp + 4 * i); |
564 | writel(val: 0, addr: ctlr->params.rxcp + 4 * i); |
565 | } |
566 | |
567 | dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff); |
568 | dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff); |
569 | |
570 | dma_reg_write(ctlr, CPDMA_TXCONTROL, 1); |
571 | dma_reg_write(ctlr, CPDMA_RXCONTROL, 1); |
572 | |
573 | ctlr->state = CPDMA_STATE_ACTIVE; |
574 | |
575 | prio_mode = 0; |
576 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { |
577 | chan = ctlr->channels[i]; |
578 | if (chan) { |
579 | cpdma_chan_set_chan_shaper(chan); |
580 | cpdma_chan_on(chan); |
581 | |
582 | /* off prio mode if all tx channels are rate limited */ |
583 | if (is_tx_chan(chan) && !chan->rate) |
584 | prio_mode = 1; |
585 | } |
586 | } |
587 | |
588 | _cpdma_control_set(ctlr, control: CPDMA_TX_PRIO_FIXED, value: prio_mode); |
589 | _cpdma_control_set(ctlr, control: CPDMA_RX_BUFFER_OFFSET, value: 0); |
590 | |
591 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
592 | return 0; |
593 | } |
594 | |
595 | int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr) |
596 | { |
597 | unsigned long flags; |
598 | int i; |
599 | |
600 | spin_lock_irqsave(&ctlr->lock, flags); |
601 | if (ctlr->state != CPDMA_STATE_ACTIVE) { |
602 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
603 | return -EINVAL; |
604 | } |
605 | |
606 | ctlr->state = CPDMA_STATE_TEARDOWN; |
607 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
608 | |
609 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { |
610 | if (ctlr->channels[i]) |
611 | cpdma_chan_stop(chan: ctlr->channels[i]); |
612 | } |
613 | |
614 | spin_lock_irqsave(&ctlr->lock, flags); |
615 | dma_reg_write(ctlr, CPDMA_RXINTMASKCLEAR, 0xffffffff); |
616 | dma_reg_write(ctlr, CPDMA_TXINTMASKCLEAR, 0xffffffff); |
617 | |
618 | dma_reg_write(ctlr, CPDMA_TXCONTROL, 0); |
619 | dma_reg_write(ctlr, CPDMA_RXCONTROL, 0); |
620 | |
621 | ctlr->state = CPDMA_STATE_IDLE; |
622 | |
623 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
624 | return 0; |
625 | } |
626 | |
627 | int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr) |
628 | { |
629 | int ret = 0, i; |
630 | |
631 | if (!ctlr) |
632 | return -EINVAL; |
633 | |
634 | if (ctlr->state != CPDMA_STATE_IDLE) |
635 | cpdma_ctlr_stop(ctlr); |
636 | |
637 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) |
638 | cpdma_chan_destroy(chan: ctlr->channels[i]); |
639 | |
640 | cpdma_desc_pool_destroy(ctlr); |
641 | return ret; |
642 | } |
643 | |
644 | int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable) |
645 | { |
646 | unsigned long flags; |
647 | int i; |
648 | |
649 | spin_lock_irqsave(&ctlr->lock, flags); |
650 | if (ctlr->state != CPDMA_STATE_ACTIVE) { |
651 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
652 | return -EINVAL; |
653 | } |
654 | |
655 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { |
656 | if (ctlr->channels[i]) |
657 | cpdma_chan_int_ctrl(chan: ctlr->channels[i], enable); |
658 | } |
659 | |
660 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
661 | return 0; |
662 | } |
663 | |
664 | void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value) |
665 | { |
666 | dma_reg_write(ctlr, CPDMA_MACEOIVECTOR, value); |
667 | } |
668 | |
669 | u32 cpdma_ctrl_rxchs_state(struct cpdma_ctlr *ctlr) |
670 | { |
671 | return dma_reg_read(ctlr, CPDMA_RXINTSTATMASKED); |
672 | } |
673 | |
674 | u32 cpdma_ctrl_txchs_state(struct cpdma_ctlr *ctlr) |
675 | { |
676 | return dma_reg_read(ctlr, CPDMA_TXINTSTATMASKED); |
677 | } |
678 | |
679 | static void cpdma_chan_set_descs(struct cpdma_ctlr *ctlr, |
680 | int rx, int desc_num, |
681 | int per_ch_desc) |
682 | { |
683 | struct cpdma_chan *chan, *most_chan = NULL; |
684 | int desc_cnt = desc_num; |
685 | int most_dnum = 0; |
686 | int min, max, i; |
687 | |
688 | if (!desc_num) |
689 | return; |
690 | |
691 | if (rx) { |
692 | min = rx_chan_num(0); |
693 | max = rx_chan_num(CPDMA_MAX_CHANNELS); |
694 | } else { |
695 | min = tx_chan_num(0); |
696 | max = tx_chan_num(CPDMA_MAX_CHANNELS); |
697 | } |
698 | |
699 | for (i = min; i < max; i++) { |
700 | chan = ctlr->channels[i]; |
701 | if (!chan) |
702 | continue; |
703 | |
704 | if (chan->weight) |
705 | chan->desc_num = (chan->weight * desc_num) / 100; |
706 | else |
707 | chan->desc_num = per_ch_desc; |
708 | |
709 | desc_cnt -= chan->desc_num; |
710 | |
711 | if (most_dnum < chan->desc_num) { |
712 | most_dnum = chan->desc_num; |
713 | most_chan = chan; |
714 | } |
715 | } |
716 | /* use remains */ |
717 | if (most_chan) |
718 | most_chan->desc_num += desc_cnt; |
719 | } |
720 | |
721 | /* |
722 | * cpdma_chan_split_pool - Splits ctrl pool between all channels. |
723 | * Has to be called under ctlr lock |
724 | */ |
725 | static int cpdma_chan_split_pool(struct cpdma_ctlr *ctlr) |
726 | { |
727 | int tx_per_ch_desc = 0, rx_per_ch_desc = 0; |
728 | int free_rx_num = 0, free_tx_num = 0; |
729 | int rx_weight = 0, tx_weight = 0; |
730 | int tx_desc_num, rx_desc_num; |
731 | struct cpdma_chan *chan; |
732 | int i; |
733 | |
734 | if (!ctlr->chan_num) |
735 | return 0; |
736 | |
737 | for (i = 0; i < ARRAY_SIZE(ctlr->channels); i++) { |
738 | chan = ctlr->channels[i]; |
739 | if (!chan) |
740 | continue; |
741 | |
742 | if (is_rx_chan(chan)) { |
743 | if (!chan->weight) |
744 | free_rx_num++; |
745 | rx_weight += chan->weight; |
746 | } else { |
747 | if (!chan->weight) |
748 | free_tx_num++; |
749 | tx_weight += chan->weight; |
750 | } |
751 | } |
752 | |
753 | if (rx_weight > 100 || tx_weight > 100) |
754 | return -EINVAL; |
755 | |
756 | tx_desc_num = ctlr->num_tx_desc; |
757 | rx_desc_num = ctlr->num_rx_desc; |
758 | |
759 | if (free_tx_num) { |
760 | tx_per_ch_desc = tx_desc_num - (tx_weight * tx_desc_num) / 100; |
761 | tx_per_ch_desc /= free_tx_num; |
762 | } |
763 | if (free_rx_num) { |
764 | rx_per_ch_desc = rx_desc_num - (rx_weight * rx_desc_num) / 100; |
765 | rx_per_ch_desc /= free_rx_num; |
766 | } |
767 | |
768 | cpdma_chan_set_descs(ctlr, rx: 0, desc_num: tx_desc_num, per_ch_desc: tx_per_ch_desc); |
769 | cpdma_chan_set_descs(ctlr, rx: 1, desc_num: rx_desc_num, per_ch_desc: rx_per_ch_desc); |
770 | |
771 | return 0; |
772 | } |
773 | |
774 | |
775 | /* cpdma_chan_set_weight - set weight of a channel in percentage. |
776 | * Tx and Rx channels have separate weights. That is 100% for RX |
777 | * and 100% for Tx. The weight is used to split cpdma resources |
778 | * in correct proportion required by the channels, including number |
779 | * of descriptors. The channel rate is not enough to know the |
780 | * weight of a channel as the maximum rate of an interface is needed. |
781 | * If weight = 0, then channel uses rest of descriptors leaved by |
782 | * weighted channels. |
783 | */ |
784 | int cpdma_chan_set_weight(struct cpdma_chan *ch, int weight) |
785 | { |
786 | struct cpdma_ctlr *ctlr = ch->ctlr; |
787 | unsigned long flags, ch_flags; |
788 | int ret; |
789 | |
790 | spin_lock_irqsave(&ctlr->lock, flags); |
791 | spin_lock_irqsave(&ch->lock, ch_flags); |
792 | if (ch->weight == weight) { |
793 | spin_unlock_irqrestore(lock: &ch->lock, flags: ch_flags); |
794 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
795 | return 0; |
796 | } |
797 | ch->weight = weight; |
798 | spin_unlock_irqrestore(lock: &ch->lock, flags: ch_flags); |
799 | |
800 | /* re-split pool using new channel weight */ |
801 | ret = cpdma_chan_split_pool(ctlr); |
802 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
803 | return ret; |
804 | } |
805 | |
806 | /* cpdma_chan_get_min_rate - get minimum allowed rate for channel |
807 | * Should be called before cpdma_chan_set_rate. |
808 | * Returns min rate in Kb/s |
809 | */ |
810 | u32 cpdma_chan_get_min_rate(struct cpdma_ctlr *ctlr) |
811 | { |
812 | unsigned int divident, divisor; |
813 | |
814 | divident = ctlr->params.bus_freq_mhz * 32 * 1000; |
815 | divisor = 1 + CPDMA_MAX_RLIM_CNT; |
816 | |
817 | return DIV_ROUND_UP(divident, divisor); |
818 | } |
819 | |
820 | /* cpdma_chan_set_rate - limits bandwidth for transmit channel. |
821 | * The bandwidth * limited channels have to be in order beginning from lowest. |
822 | * ch - transmit channel the bandwidth is configured for |
823 | * rate - bandwidth in Kb/s, if 0 - then off shaper |
824 | */ |
825 | int cpdma_chan_set_rate(struct cpdma_chan *ch, u32 rate) |
826 | { |
827 | unsigned long flags, ch_flags; |
828 | struct cpdma_ctlr *ctlr; |
829 | int ret, prio_mode; |
830 | u32 rmask; |
831 | |
832 | if (!ch || !is_tx_chan(ch)) |
833 | return -EINVAL; |
834 | |
835 | if (ch->rate == rate) |
836 | return rate; |
837 | |
838 | ctlr = ch->ctlr; |
839 | spin_lock_irqsave(&ctlr->lock, flags); |
840 | spin_lock_irqsave(&ch->lock, ch_flags); |
841 | |
842 | ret = cpdma_chan_fit_rate(ch, rate, rmask: &rmask, prio_mode: &prio_mode); |
843 | if (ret) |
844 | goto err; |
845 | |
846 | ret = cpdma_chan_set_factors(ctlr, ch); |
847 | if (ret) |
848 | goto err; |
849 | |
850 | spin_unlock_irqrestore(lock: &ch->lock, flags: ch_flags); |
851 | |
852 | /* on shapers */ |
853 | _cpdma_control_set(ctlr, control: CPDMA_TX_RLIM, value: rmask); |
854 | _cpdma_control_set(ctlr, control: CPDMA_TX_PRIO_FIXED, value: prio_mode); |
855 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
856 | return ret; |
857 | |
858 | err: |
859 | spin_unlock_irqrestore(lock: &ch->lock, flags: ch_flags); |
860 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
861 | return ret; |
862 | } |
863 | |
864 | u32 cpdma_chan_get_rate(struct cpdma_chan *ch) |
865 | { |
866 | unsigned long flags; |
867 | u32 rate; |
868 | |
869 | spin_lock_irqsave(&ch->lock, flags); |
870 | rate = ch->rate; |
871 | spin_unlock_irqrestore(lock: &ch->lock, flags); |
872 | |
873 | return rate; |
874 | } |
875 | |
876 | struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num, |
877 | cpdma_handler_fn handler, int rx_type) |
878 | { |
879 | int offset = chan_num * 4; |
880 | struct cpdma_chan *chan; |
881 | unsigned long flags; |
882 | |
883 | chan_num = rx_type ? rx_chan_num(chan_num) : tx_chan_num(chan_num); |
884 | |
885 | if (__chan_linear(chan_num) >= ctlr->num_chan) |
886 | return ERR_PTR(error: -EINVAL); |
887 | |
888 | chan = devm_kzalloc(dev: ctlr->dev, size: sizeof(*chan), GFP_KERNEL); |
889 | if (!chan) |
890 | return ERR_PTR(error: -ENOMEM); |
891 | |
892 | spin_lock_irqsave(&ctlr->lock, flags); |
893 | if (ctlr->channels[chan_num]) { |
894 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
895 | devm_kfree(dev: ctlr->dev, p: chan); |
896 | return ERR_PTR(error: -EBUSY); |
897 | } |
898 | |
899 | chan->ctlr = ctlr; |
900 | chan->state = CPDMA_STATE_IDLE; |
901 | chan->chan_num = chan_num; |
902 | chan->handler = handler; |
903 | chan->rate = 0; |
904 | chan->weight = 0; |
905 | |
906 | if (is_rx_chan(chan)) { |
907 | chan->hdp = ctlr->params.rxhdp + offset; |
908 | chan->cp = ctlr->params.rxcp + offset; |
909 | chan->rxfree = ctlr->params.rxfree + offset; |
910 | chan->int_set = CPDMA_RXINTMASKSET; |
911 | chan->int_clear = CPDMA_RXINTMASKCLEAR; |
912 | chan->td = CPDMA_RXTEARDOWN; |
913 | chan->dir = DMA_FROM_DEVICE; |
914 | } else { |
915 | chan->hdp = ctlr->params.txhdp + offset; |
916 | chan->cp = ctlr->params.txcp + offset; |
917 | chan->int_set = CPDMA_TXINTMASKSET; |
918 | chan->int_clear = CPDMA_TXINTMASKCLEAR; |
919 | chan->td = CPDMA_TXTEARDOWN; |
920 | chan->dir = DMA_TO_DEVICE; |
921 | } |
922 | chan->mask = BIT(chan_linear(chan)); |
923 | |
924 | spin_lock_init(&chan->lock); |
925 | |
926 | ctlr->channels[chan_num] = chan; |
927 | ctlr->chan_num++; |
928 | |
929 | cpdma_chan_split_pool(ctlr); |
930 | |
931 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
932 | return chan; |
933 | } |
934 | |
935 | int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan) |
936 | { |
937 | unsigned long flags; |
938 | int desc_num; |
939 | |
940 | spin_lock_irqsave(&chan->lock, flags); |
941 | desc_num = chan->desc_num; |
942 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
943 | |
944 | return desc_num; |
945 | } |
946 | |
947 | int cpdma_chan_destroy(struct cpdma_chan *chan) |
948 | { |
949 | struct cpdma_ctlr *ctlr; |
950 | unsigned long flags; |
951 | |
952 | if (!chan) |
953 | return -EINVAL; |
954 | ctlr = chan->ctlr; |
955 | |
956 | spin_lock_irqsave(&ctlr->lock, flags); |
957 | if (chan->state != CPDMA_STATE_IDLE) |
958 | cpdma_chan_stop(chan); |
959 | ctlr->channels[chan->chan_num] = NULL; |
960 | ctlr->chan_num--; |
961 | devm_kfree(dev: ctlr->dev, p: chan); |
962 | cpdma_chan_split_pool(ctlr); |
963 | |
964 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
965 | return 0; |
966 | } |
967 | |
968 | int cpdma_chan_get_stats(struct cpdma_chan *chan, |
969 | struct cpdma_chan_stats *stats) |
970 | { |
971 | unsigned long flags; |
972 | if (!chan) |
973 | return -EINVAL; |
974 | spin_lock_irqsave(&chan->lock, flags); |
975 | memcpy(stats, &chan->stats, sizeof(*stats)); |
976 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
977 | return 0; |
978 | } |
979 | |
980 | static void __cpdma_chan_submit(struct cpdma_chan *chan, |
981 | struct cpdma_desc __iomem *desc) |
982 | { |
983 | struct cpdma_ctlr *ctlr = chan->ctlr; |
984 | struct cpdma_desc __iomem *prev = chan->tail; |
985 | struct cpdma_desc_pool *pool = ctlr->pool; |
986 | dma_addr_t desc_dma; |
987 | u32 mode; |
988 | |
989 | desc_dma = desc_phys(pool, desc); |
990 | |
991 | /* simple case - idle channel */ |
992 | if (!chan->head) { |
993 | chan->stats.head_enqueue++; |
994 | chan->head = desc; |
995 | chan->tail = desc; |
996 | if (chan->state == CPDMA_STATE_ACTIVE) |
997 | chan_write(chan, hdp, desc_dma); |
998 | return; |
999 | } |
1000 | |
1001 | /* first chain the descriptor at the tail of the list */ |
1002 | desc_write(prev, hw_next, desc_dma); |
1003 | chan->tail = desc; |
1004 | chan->stats.tail_enqueue++; |
1005 | |
1006 | /* next check if EOQ has been triggered already */ |
1007 | mode = desc_read(prev, hw_mode); |
1008 | if (((mode & (CPDMA_DESC_EOQ | CPDMA_DESC_OWNER)) == CPDMA_DESC_EOQ) && |
1009 | (chan->state == CPDMA_STATE_ACTIVE)) { |
1010 | desc_write(prev, hw_mode, mode & ~CPDMA_DESC_EOQ); |
1011 | chan_write(chan, hdp, desc_dma); |
1012 | chan->stats.misqueued++; |
1013 | } |
1014 | } |
1015 | |
1016 | static int cpdma_chan_submit_si(struct submit_info *si) |
1017 | { |
1018 | struct cpdma_chan *chan = si->chan; |
1019 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1020 | int len = si->len; |
1021 | struct cpdma_desc __iomem *desc; |
1022 | dma_addr_t buffer; |
1023 | u32 mode; |
1024 | int ret; |
1025 | |
1026 | if (chan->count >= chan->desc_num) { |
1027 | chan->stats.desc_alloc_fail++; |
1028 | return -ENOMEM; |
1029 | } |
1030 | |
1031 | desc = cpdma_desc_alloc(pool: ctlr->pool); |
1032 | if (!desc) { |
1033 | chan->stats.desc_alloc_fail++; |
1034 | return -ENOMEM; |
1035 | } |
1036 | |
1037 | if (len < ctlr->params.min_packet_size) { |
1038 | len = ctlr->params.min_packet_size; |
1039 | chan->stats.runt_transmit_buff++; |
1040 | } |
1041 | |
1042 | mode = CPDMA_DESC_OWNER | CPDMA_DESC_SOP | CPDMA_DESC_EOP; |
1043 | cpdma_desc_to_port(chan, mode, si->directed); |
1044 | |
1045 | if (si->data_dma) { |
1046 | buffer = si->data_dma; |
1047 | dma_sync_single_for_device(dev: ctlr->dev, addr: buffer, size: len, dir: chan->dir); |
1048 | } else { |
1049 | buffer = dma_map_single(ctlr->dev, si->data_virt, len, chan->dir); |
1050 | ret = dma_mapping_error(dev: ctlr->dev, dma_addr: buffer); |
1051 | if (ret) { |
1052 | cpdma_desc_free(pool: ctlr->pool, desc, num_desc: 1); |
1053 | return -EINVAL; |
1054 | } |
1055 | } |
1056 | |
1057 | /* Relaxed IO accessors can be used here as there is read barrier |
1058 | * at the end of write sequence. |
1059 | */ |
1060 | writel_relaxed(0, &desc->hw_next); |
1061 | writel_relaxed(buffer, &desc->hw_buffer); |
1062 | writel_relaxed(len, &desc->hw_len); |
1063 | writel_relaxed(mode | len, &desc->hw_mode); |
1064 | writel_relaxed((uintptr_t)si->token, &desc->sw_token); |
1065 | writel_relaxed(buffer, &desc->sw_buffer); |
1066 | writel_relaxed(si->data_dma ? len | CPDMA_DMA_EXT_MAP : len, |
1067 | &desc->sw_len); |
1068 | desc_read(desc, sw_len); |
1069 | |
1070 | __cpdma_chan_submit(chan, desc); |
1071 | |
1072 | if (chan->state == CPDMA_STATE_ACTIVE && chan->rxfree) |
1073 | chan_write(chan, rxfree, 1); |
1074 | |
1075 | chan->count++; |
1076 | return 0; |
1077 | } |
1078 | |
1079 | int cpdma_chan_idle_submit(struct cpdma_chan *chan, void *token, void *data, |
1080 | int len, int directed) |
1081 | { |
1082 | struct submit_info si; |
1083 | unsigned long flags; |
1084 | int ret; |
1085 | |
1086 | si.chan = chan; |
1087 | si.token = token; |
1088 | si.data_virt = data; |
1089 | si.data_dma = 0; |
1090 | si.len = len; |
1091 | si.directed = directed; |
1092 | |
1093 | spin_lock_irqsave(&chan->lock, flags); |
1094 | if (chan->state == CPDMA_STATE_TEARDOWN) { |
1095 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1096 | return -EINVAL; |
1097 | } |
1098 | |
1099 | ret = cpdma_chan_submit_si(si: &si); |
1100 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1101 | return ret; |
1102 | } |
1103 | |
1104 | int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token, |
1105 | dma_addr_t data, int len, int directed) |
1106 | { |
1107 | struct submit_info si; |
1108 | unsigned long flags; |
1109 | int ret; |
1110 | |
1111 | si.chan = chan; |
1112 | si.token = token; |
1113 | si.data_virt = NULL; |
1114 | si.data_dma = data; |
1115 | si.len = len; |
1116 | si.directed = directed; |
1117 | |
1118 | spin_lock_irqsave(&chan->lock, flags); |
1119 | if (chan->state == CPDMA_STATE_TEARDOWN) { |
1120 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1121 | return -EINVAL; |
1122 | } |
1123 | |
1124 | ret = cpdma_chan_submit_si(si: &si); |
1125 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1126 | return ret; |
1127 | } |
1128 | |
1129 | int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data, |
1130 | int len, int directed) |
1131 | { |
1132 | struct submit_info si; |
1133 | unsigned long flags; |
1134 | int ret; |
1135 | |
1136 | si.chan = chan; |
1137 | si.token = token; |
1138 | si.data_virt = data; |
1139 | si.data_dma = 0; |
1140 | si.len = len; |
1141 | si.directed = directed; |
1142 | |
1143 | spin_lock_irqsave(&chan->lock, flags); |
1144 | if (chan->state != CPDMA_STATE_ACTIVE) { |
1145 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1146 | return -EINVAL; |
1147 | } |
1148 | |
1149 | ret = cpdma_chan_submit_si(si: &si); |
1150 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1151 | return ret; |
1152 | } |
1153 | |
1154 | int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token, |
1155 | dma_addr_t data, int len, int directed) |
1156 | { |
1157 | struct submit_info si; |
1158 | unsigned long flags; |
1159 | int ret; |
1160 | |
1161 | si.chan = chan; |
1162 | si.token = token; |
1163 | si.data_virt = NULL; |
1164 | si.data_dma = data; |
1165 | si.len = len; |
1166 | si.directed = directed; |
1167 | |
1168 | spin_lock_irqsave(&chan->lock, flags); |
1169 | if (chan->state != CPDMA_STATE_ACTIVE) { |
1170 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1171 | return -EINVAL; |
1172 | } |
1173 | |
1174 | ret = cpdma_chan_submit_si(si: &si); |
1175 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1176 | return ret; |
1177 | } |
1178 | |
1179 | bool cpdma_check_free_tx_desc(struct cpdma_chan *chan) |
1180 | { |
1181 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1182 | struct cpdma_desc_pool *pool = ctlr->pool; |
1183 | bool free_tx_desc; |
1184 | unsigned long flags; |
1185 | |
1186 | spin_lock_irqsave(&chan->lock, flags); |
1187 | free_tx_desc = (chan->count < chan->desc_num) && |
1188 | gen_pool_avail(pool->gen_pool); |
1189 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1190 | return free_tx_desc; |
1191 | } |
1192 | |
1193 | static void __cpdma_chan_free(struct cpdma_chan *chan, |
1194 | struct cpdma_desc __iomem *desc, |
1195 | int outlen, int status) |
1196 | { |
1197 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1198 | struct cpdma_desc_pool *pool = ctlr->pool; |
1199 | dma_addr_t buff_dma; |
1200 | int origlen; |
1201 | uintptr_t token; |
1202 | |
1203 | token = desc_read(desc, sw_token); |
1204 | origlen = desc_read(desc, sw_len); |
1205 | |
1206 | buff_dma = desc_read(desc, sw_buffer); |
1207 | if (origlen & CPDMA_DMA_EXT_MAP) { |
1208 | origlen &= ~CPDMA_DMA_EXT_MAP; |
1209 | dma_sync_single_for_cpu(dev: ctlr->dev, addr: buff_dma, size: origlen, |
1210 | dir: chan->dir); |
1211 | } else { |
1212 | dma_unmap_single(ctlr->dev, buff_dma, origlen, chan->dir); |
1213 | } |
1214 | |
1215 | cpdma_desc_free(pool, desc, num_desc: 1); |
1216 | (*chan->handler)((void *)token, outlen, status); |
1217 | } |
1218 | |
1219 | static int __cpdma_chan_process(struct cpdma_chan *chan) |
1220 | { |
1221 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1222 | struct cpdma_desc __iomem *desc; |
1223 | int status, outlen; |
1224 | int cb_status = 0; |
1225 | struct cpdma_desc_pool *pool = ctlr->pool; |
1226 | dma_addr_t desc_dma; |
1227 | unsigned long flags; |
1228 | |
1229 | spin_lock_irqsave(&chan->lock, flags); |
1230 | |
1231 | desc = chan->head; |
1232 | if (!desc) { |
1233 | chan->stats.empty_dequeue++; |
1234 | status = -ENOENT; |
1235 | goto unlock_ret; |
1236 | } |
1237 | desc_dma = desc_phys(pool, desc); |
1238 | |
1239 | status = desc_read(desc, hw_mode); |
1240 | outlen = status & 0x7ff; |
1241 | if (status & CPDMA_DESC_OWNER) { |
1242 | chan->stats.busy_dequeue++; |
1243 | status = -EBUSY; |
1244 | goto unlock_ret; |
1245 | } |
1246 | |
1247 | if (status & CPDMA_DESC_PASS_CRC) |
1248 | outlen -= CPDMA_DESC_CRC_LEN; |
1249 | |
1250 | status = status & (CPDMA_DESC_EOQ | CPDMA_DESC_TD_COMPLETE | |
1251 | CPDMA_DESC_PORT_MASK | CPDMA_RX_VLAN_ENCAP); |
1252 | |
1253 | chan->head = desc_from_phys(pool, desc_read(desc, hw_next)); |
1254 | chan_write(chan, cp, desc_dma); |
1255 | chan->count--; |
1256 | chan->stats.good_dequeue++; |
1257 | |
1258 | if ((status & CPDMA_DESC_EOQ) && chan->head) { |
1259 | chan->stats.requeue++; |
1260 | chan_write(chan, hdp, desc_phys(pool, chan->head)); |
1261 | } |
1262 | |
1263 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1264 | if (unlikely(status & CPDMA_DESC_TD_COMPLETE)) |
1265 | cb_status = -ENOSYS; |
1266 | else |
1267 | cb_status = status; |
1268 | |
1269 | __cpdma_chan_free(chan, desc, outlen, status: cb_status); |
1270 | return status; |
1271 | |
1272 | unlock_ret: |
1273 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1274 | return status; |
1275 | } |
1276 | |
1277 | int cpdma_chan_process(struct cpdma_chan *chan, int quota) |
1278 | { |
1279 | int used = 0, ret = 0; |
1280 | |
1281 | if (chan->state != CPDMA_STATE_ACTIVE) |
1282 | return -EINVAL; |
1283 | |
1284 | while (used < quota) { |
1285 | ret = __cpdma_chan_process(chan); |
1286 | if (ret < 0) |
1287 | break; |
1288 | used++; |
1289 | } |
1290 | return used; |
1291 | } |
1292 | |
1293 | int cpdma_chan_start(struct cpdma_chan *chan) |
1294 | { |
1295 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1296 | unsigned long flags; |
1297 | int ret; |
1298 | |
1299 | spin_lock_irqsave(&ctlr->lock, flags); |
1300 | ret = cpdma_chan_set_chan_shaper(chan); |
1301 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
1302 | if (ret) |
1303 | return ret; |
1304 | |
1305 | ret = cpdma_chan_on(chan); |
1306 | if (ret) |
1307 | return ret; |
1308 | |
1309 | return 0; |
1310 | } |
1311 | |
1312 | int cpdma_chan_stop(struct cpdma_chan *chan) |
1313 | { |
1314 | struct cpdma_ctlr *ctlr = chan->ctlr; |
1315 | struct cpdma_desc_pool *pool = ctlr->pool; |
1316 | unsigned long flags; |
1317 | int ret; |
1318 | unsigned timeout; |
1319 | |
1320 | spin_lock_irqsave(&chan->lock, flags); |
1321 | if (chan->state == CPDMA_STATE_TEARDOWN) { |
1322 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1323 | return -EINVAL; |
1324 | } |
1325 | |
1326 | chan->state = CPDMA_STATE_TEARDOWN; |
1327 | dma_reg_write(ctlr, chan->int_clear, chan->mask); |
1328 | |
1329 | /* trigger teardown */ |
1330 | dma_reg_write(ctlr, chan->td, chan_linear(chan)); |
1331 | |
1332 | /* wait for teardown complete */ |
1333 | timeout = 100 * 100; /* 100 ms */ |
1334 | while (timeout) { |
1335 | u32 cp = chan_read(chan, cp); |
1336 | if ((cp & CPDMA_TEARDOWN_VALUE) == CPDMA_TEARDOWN_VALUE) |
1337 | break; |
1338 | udelay(10); |
1339 | timeout--; |
1340 | } |
1341 | WARN_ON(!timeout); |
1342 | chan_write(chan, cp, CPDMA_TEARDOWN_VALUE); |
1343 | |
1344 | /* handle completed packets */ |
1345 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1346 | do { |
1347 | ret = __cpdma_chan_process(chan); |
1348 | if (ret < 0) |
1349 | break; |
1350 | } while ((ret & CPDMA_DESC_TD_COMPLETE) == 0); |
1351 | spin_lock_irqsave(&chan->lock, flags); |
1352 | |
1353 | /* remaining packets haven't been tx/rx'ed, clean them up */ |
1354 | while (chan->head) { |
1355 | struct cpdma_desc __iomem *desc = chan->head; |
1356 | dma_addr_t next_dma; |
1357 | |
1358 | next_dma = desc_read(desc, hw_next); |
1359 | chan->head = desc_from_phys(pool, dma: next_dma); |
1360 | chan->count--; |
1361 | chan->stats.teardown_dequeue++; |
1362 | |
1363 | /* issue callback without locks held */ |
1364 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1365 | __cpdma_chan_free(chan, desc, outlen: 0, status: -ENOSYS); |
1366 | spin_lock_irqsave(&chan->lock, flags); |
1367 | } |
1368 | |
1369 | chan->state = CPDMA_STATE_IDLE; |
1370 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1371 | return 0; |
1372 | } |
1373 | |
1374 | int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable) |
1375 | { |
1376 | unsigned long flags; |
1377 | |
1378 | spin_lock_irqsave(&chan->lock, flags); |
1379 | if (chan->state != CPDMA_STATE_ACTIVE) { |
1380 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1381 | return -EINVAL; |
1382 | } |
1383 | |
1384 | dma_reg_write(chan->ctlr, enable ? chan->int_set : chan->int_clear, |
1385 | chan->mask); |
1386 | spin_unlock_irqrestore(lock: &chan->lock, flags); |
1387 | |
1388 | return 0; |
1389 | } |
1390 | |
1391 | int cpdma_control_get(struct cpdma_ctlr *ctlr, int control) |
1392 | { |
1393 | unsigned long flags; |
1394 | int ret; |
1395 | |
1396 | spin_lock_irqsave(&ctlr->lock, flags); |
1397 | ret = _cpdma_control_get(ctlr, control); |
1398 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
1399 | |
1400 | return ret; |
1401 | } |
1402 | |
1403 | int cpdma_control_set(struct cpdma_ctlr *ctlr, int control, int value) |
1404 | { |
1405 | unsigned long flags; |
1406 | int ret; |
1407 | |
1408 | spin_lock_irqsave(&ctlr->lock, flags); |
1409 | ret = _cpdma_control_set(ctlr, control, value); |
1410 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
1411 | |
1412 | return ret; |
1413 | } |
1414 | |
1415 | int cpdma_get_num_rx_descs(struct cpdma_ctlr *ctlr) |
1416 | { |
1417 | return ctlr->num_rx_desc; |
1418 | } |
1419 | |
1420 | int cpdma_get_num_tx_descs(struct cpdma_ctlr *ctlr) |
1421 | { |
1422 | return ctlr->num_tx_desc; |
1423 | } |
1424 | |
1425 | int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc) |
1426 | { |
1427 | unsigned long flags; |
1428 | int temp, ret; |
1429 | |
1430 | spin_lock_irqsave(&ctlr->lock, flags); |
1431 | |
1432 | temp = ctlr->num_rx_desc; |
1433 | ctlr->num_rx_desc = num_rx_desc; |
1434 | ctlr->num_tx_desc = ctlr->pool->num_desc - ctlr->num_rx_desc; |
1435 | ret = cpdma_chan_split_pool(ctlr); |
1436 | if (ret) { |
1437 | ctlr->num_rx_desc = temp; |
1438 | ctlr->num_tx_desc = ctlr->pool->num_desc - ctlr->num_rx_desc; |
1439 | } |
1440 | |
1441 | spin_unlock_irqrestore(lock: &ctlr->lock, flags); |
1442 | |
1443 | return ret; |
1444 | } |
1445 | |