1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * ADMA driver for Nvidia's Tegra210 ADMA controller. |
4 | * |
5 | * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. |
6 | */ |
7 | |
8 | #include <linux/clk.h> |
9 | #include <linux/iopoll.h> |
10 | #include <linux/module.h> |
11 | #include <linux/of.h> |
12 | #include <linux/of_dma.h> |
13 | #include <linux/of_irq.h> |
14 | #include <linux/platform_device.h> |
15 | #include <linux/pm_runtime.h> |
16 | #include <linux/slab.h> |
17 | |
18 | #include "virt-dma.h" |
19 | |
20 | #define ADMA_CH_CMD 0x00 |
21 | #define ADMA_CH_STATUS 0x0c |
22 | #define ADMA_CH_STATUS_XFER_EN BIT(0) |
23 | #define ADMA_CH_STATUS_XFER_PAUSED BIT(1) |
24 | |
25 | #define ADMA_CH_INT_STATUS 0x10 |
26 | #define ADMA_CH_INT_STATUS_XFER_DONE BIT(0) |
27 | |
28 | #define ADMA_CH_INT_CLEAR 0x1c |
29 | #define ADMA_CH_CTRL 0x24 |
30 | #define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12) |
31 | #define ADMA_CH_CTRL_DIR_AHUB2MEM 2 |
32 | #define ADMA_CH_CTRL_DIR_MEM2AHUB 4 |
33 | #define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8) |
34 | #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1) |
35 | #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0 |
36 | |
37 | #define ADMA_CH_CONFIG 0x28 |
38 | #define ADMA_CH_CONFIG_SRC_BUF(val) (((val) & 0x7) << 28) |
39 | #define ADMA_CH_CONFIG_TRG_BUF(val) (((val) & 0x7) << 24) |
40 | #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT 20 |
41 | #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16 |
42 | #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf) |
43 | #define ADMA_CH_CONFIG_MAX_BUFS 8 |
44 | #define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4) |
45 | |
46 | #define ADMA_CH_FIFO_CTRL 0x2c |
47 | #define ADMA_CH_TX_FIFO_SIZE_SHIFT 8 |
48 | #define ADMA_CH_RX_FIFO_SIZE_SHIFT 0 |
49 | |
50 | #define ADMA_CH_LOWER_SRC_ADDR 0x34 |
51 | #define ADMA_CH_LOWER_TRG_ADDR 0x3c |
52 | #define ADMA_CH_TC 0x44 |
53 | #define ADMA_CH_TC_COUNT_MASK 0x3ffffffc |
54 | |
55 | #define ADMA_CH_XFER_STATUS 0x54 |
56 | #define ADMA_CH_XFER_STATUS_COUNT_MASK 0xffff |
57 | |
58 | #define ADMA_GLOBAL_CMD 0x00 |
59 | #define ADMA_GLOBAL_SOFT_RESET 0x04 |
60 | |
61 | #define TEGRA_ADMA_BURST_COMPLETE_TIME 20 |
62 | |
63 | #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift) |
64 | |
65 | struct tegra_adma; |
66 | |
67 | /* |
68 | * struct tegra_adma_chip_data - Tegra chip specific data |
69 | * @adma_get_burst_config: Function callback used to set DMA burst size. |
70 | * @global_reg_offset: Register offset of DMA global register. |
71 | * @global_int_clear: Register offset of DMA global interrupt clear. |
72 | * @ch_req_tx_shift: Register offset for AHUB transmit channel select. |
73 | * @ch_req_rx_shift: Register offset for AHUB receive channel select. |
74 | * @ch_base_offset: Register offset of DMA channel registers. |
75 | * @ch_fifo_ctrl: Default value for channel FIFO CTRL register. |
76 | * @ch_req_mask: Mask for Tx or Rx channel select. |
77 | * @ch_req_max: Maximum number of Tx or Rx channels available. |
78 | * @ch_reg_size: Size of DMA channel register space. |
79 | * @nr_channels: Number of DMA channels available. |
80 | * @ch_fifo_size_mask: Mask for FIFO size field. |
81 | * @sreq_index_offset: Slave channel index offset. |
82 | * @has_outstanding_reqs: If DMA channel can have outstanding requests. |
83 | */ |
84 | struct tegra_adma_chip_data { |
85 | unsigned int (*adma_get_burst_config)(unsigned int burst_size); |
86 | unsigned int global_reg_offset; |
87 | unsigned int global_int_clear; |
88 | unsigned int ch_req_tx_shift; |
89 | unsigned int ch_req_rx_shift; |
90 | unsigned int ch_base_offset; |
91 | unsigned int ch_fifo_ctrl; |
92 | unsigned int ch_req_mask; |
93 | unsigned int ch_req_max; |
94 | unsigned int ch_reg_size; |
95 | unsigned int nr_channels; |
96 | unsigned int ch_fifo_size_mask; |
97 | unsigned int sreq_index_offset; |
98 | bool has_outstanding_reqs; |
99 | }; |
100 | |
101 | /* |
102 | * struct tegra_adma_chan_regs - Tegra ADMA channel registers |
103 | */ |
104 | struct tegra_adma_chan_regs { |
105 | unsigned int ctrl; |
106 | unsigned int config; |
107 | unsigned int src_addr; |
108 | unsigned int trg_addr; |
109 | unsigned int fifo_ctrl; |
110 | unsigned int cmd; |
111 | unsigned int tc; |
112 | }; |
113 | |
114 | /* |
115 | * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests. |
116 | */ |
117 | struct tegra_adma_desc { |
118 | struct virt_dma_desc vd; |
119 | struct tegra_adma_chan_regs ch_regs; |
120 | size_t buf_len; |
121 | size_t period_len; |
122 | size_t num_periods; |
123 | }; |
124 | |
125 | /* |
126 | * struct tegra_adma_chan - Tegra ADMA channel information |
127 | */ |
128 | struct tegra_adma_chan { |
129 | struct virt_dma_chan vc; |
130 | struct tegra_adma_desc *desc; |
131 | struct tegra_adma *tdma; |
132 | int irq; |
133 | void __iomem *chan_addr; |
134 | |
135 | /* Slave channel configuration info */ |
136 | struct dma_slave_config sconfig; |
137 | enum dma_transfer_direction sreq_dir; |
138 | unsigned int sreq_index; |
139 | bool sreq_reserved; |
140 | struct tegra_adma_chan_regs ch_regs; |
141 | |
142 | /* Transfer count and position info */ |
143 | unsigned int tx_buf_count; |
144 | unsigned int tx_buf_pos; |
145 | }; |
146 | |
147 | /* |
148 | * struct tegra_adma - Tegra ADMA controller information |
149 | */ |
150 | struct tegra_adma { |
151 | struct dma_device dma_dev; |
152 | struct device *dev; |
153 | void __iomem *base_addr; |
154 | struct clk *ahub_clk; |
155 | unsigned int nr_channels; |
156 | unsigned long *dma_chan_mask; |
157 | unsigned long rx_requests_reserved; |
158 | unsigned long tx_requests_reserved; |
159 | |
160 | /* Used to store global command register state when suspending */ |
161 | unsigned int global_cmd; |
162 | |
163 | const struct tegra_adma_chip_data *cdata; |
164 | |
165 | /* Last member of the structure */ |
166 | struct tegra_adma_chan channels[] __counted_by(nr_channels); |
167 | }; |
168 | |
169 | static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) |
170 | { |
171 | writel(val, addr: tdma->base_addr + tdma->cdata->global_reg_offset + reg); |
172 | } |
173 | |
174 | static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) |
175 | { |
176 | return readl(addr: tdma->base_addr + tdma->cdata->global_reg_offset + reg); |
177 | } |
178 | |
179 | static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) |
180 | { |
181 | writel(val, addr: tdc->chan_addr + reg); |
182 | } |
183 | |
184 | static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg) |
185 | { |
186 | return readl(addr: tdc->chan_addr + reg); |
187 | } |
188 | |
189 | static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc) |
190 | { |
191 | return container_of(dc, struct tegra_adma_chan, vc.chan); |
192 | } |
193 | |
194 | static inline struct tegra_adma_desc *to_tegra_adma_desc( |
195 | struct dma_async_tx_descriptor *td) |
196 | { |
197 | return container_of(td, struct tegra_adma_desc, vd.tx); |
198 | } |
199 | |
200 | static inline struct device *tdc2dev(struct tegra_adma_chan *tdc) |
201 | { |
202 | return tdc->tdma->dev; |
203 | } |
204 | |
205 | static void tegra_adma_desc_free(struct virt_dma_desc *vd) |
206 | { |
207 | kfree(container_of(vd, struct tegra_adma_desc, vd)); |
208 | } |
209 | |
210 | static int tegra_adma_slave_config(struct dma_chan *dc, |
211 | struct dma_slave_config *sconfig) |
212 | { |
213 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
214 | |
215 | memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig)); |
216 | |
217 | return 0; |
218 | } |
219 | |
220 | static int tegra_adma_init(struct tegra_adma *tdma) |
221 | { |
222 | u32 status; |
223 | int ret; |
224 | |
225 | /* Clear any interrupts */ |
226 | tdma_write(tdma, reg: tdma->cdata->ch_base_offset + tdma->cdata->global_int_clear, val: 0x1); |
227 | |
228 | /* Assert soft reset */ |
229 | tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, val: 0x1); |
230 | |
231 | /* Wait for reset to clear */ |
232 | ret = readx_poll_timeout(readl, |
233 | tdma->base_addr + |
234 | tdma->cdata->global_reg_offset + |
235 | ADMA_GLOBAL_SOFT_RESET, |
236 | status, status == 0, 20, 10000); |
237 | if (ret) |
238 | return ret; |
239 | |
240 | /* Enable global ADMA registers */ |
241 | tdma_write(tdma, ADMA_GLOBAL_CMD, val: 1); |
242 | |
243 | return 0; |
244 | } |
245 | |
246 | static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc, |
247 | enum dma_transfer_direction direction) |
248 | { |
249 | struct tegra_adma *tdma = tdc->tdma; |
250 | unsigned int sreq_index = tdc->sreq_index; |
251 | |
252 | if (tdc->sreq_reserved) |
253 | return tdc->sreq_dir == direction ? 0 : -EINVAL; |
254 | |
255 | if (sreq_index > tdma->cdata->ch_req_max) { |
256 | dev_err(tdma->dev, "invalid DMA request\n" ); |
257 | return -EINVAL; |
258 | } |
259 | |
260 | switch (direction) { |
261 | case DMA_MEM_TO_DEV: |
262 | if (test_and_set_bit(nr: sreq_index, addr: &tdma->tx_requests_reserved)) { |
263 | dev_err(tdma->dev, "DMA request reserved\n" ); |
264 | return -EINVAL; |
265 | } |
266 | break; |
267 | |
268 | case DMA_DEV_TO_MEM: |
269 | if (test_and_set_bit(nr: sreq_index, addr: &tdma->rx_requests_reserved)) { |
270 | dev_err(tdma->dev, "DMA request reserved\n" ); |
271 | return -EINVAL; |
272 | } |
273 | break; |
274 | |
275 | default: |
276 | dev_WARN(tdma->dev, "channel %s has invalid transfer type\n" , |
277 | dma_chan_name(&tdc->vc.chan)); |
278 | return -EINVAL; |
279 | } |
280 | |
281 | tdc->sreq_dir = direction; |
282 | tdc->sreq_reserved = true; |
283 | |
284 | return 0; |
285 | } |
286 | |
287 | static void tegra_adma_request_free(struct tegra_adma_chan *tdc) |
288 | { |
289 | struct tegra_adma *tdma = tdc->tdma; |
290 | |
291 | if (!tdc->sreq_reserved) |
292 | return; |
293 | |
294 | switch (tdc->sreq_dir) { |
295 | case DMA_MEM_TO_DEV: |
296 | clear_bit(nr: tdc->sreq_index, addr: &tdma->tx_requests_reserved); |
297 | break; |
298 | |
299 | case DMA_DEV_TO_MEM: |
300 | clear_bit(nr: tdc->sreq_index, addr: &tdma->rx_requests_reserved); |
301 | break; |
302 | |
303 | default: |
304 | dev_WARN(tdma->dev, "channel %s has invalid transfer type\n" , |
305 | dma_chan_name(&tdc->vc.chan)); |
306 | return; |
307 | } |
308 | |
309 | tdc->sreq_reserved = false; |
310 | } |
311 | |
312 | static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc) |
313 | { |
314 | u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS); |
315 | |
316 | return status & ADMA_CH_INT_STATUS_XFER_DONE; |
317 | } |
318 | |
319 | static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc) |
320 | { |
321 | u32 status = tegra_adma_irq_status(tdc); |
322 | |
323 | if (status) |
324 | tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, val: status); |
325 | |
326 | return status; |
327 | } |
328 | |
329 | static void tegra_adma_stop(struct tegra_adma_chan *tdc) |
330 | { |
331 | unsigned int status; |
332 | |
333 | /* Disable ADMA */ |
334 | tdma_ch_write(tdc, ADMA_CH_CMD, val: 0); |
335 | |
336 | /* Clear interrupt status */ |
337 | tegra_adma_irq_clear(tdc); |
338 | |
339 | if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS, |
340 | status, !(status & ADMA_CH_STATUS_XFER_EN), |
341 | 20, 10000)) { |
342 | dev_err(tdc2dev(tdc), "unable to stop DMA channel\n" ); |
343 | return; |
344 | } |
345 | |
346 | kfree(objp: tdc->desc); |
347 | tdc->desc = NULL; |
348 | } |
349 | |
350 | static void tegra_adma_start(struct tegra_adma_chan *tdc) |
351 | { |
352 | struct virt_dma_desc *vd = vchan_next_desc(vc: &tdc->vc); |
353 | struct tegra_adma_chan_regs *ch_regs; |
354 | struct tegra_adma_desc *desc; |
355 | |
356 | if (!vd) |
357 | return; |
358 | |
359 | list_del(entry: &vd->node); |
360 | |
361 | desc = to_tegra_adma_desc(td: &vd->tx); |
362 | |
363 | if (!desc) { |
364 | dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n" ); |
365 | return; |
366 | } |
367 | |
368 | ch_regs = &desc->ch_regs; |
369 | |
370 | tdc->tx_buf_pos = 0; |
371 | tdc->tx_buf_count = 0; |
372 | tdma_ch_write(tdc, ADMA_CH_TC, val: ch_regs->tc); |
373 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_regs->ctrl); |
374 | tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, val: ch_regs->src_addr); |
375 | tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, val: ch_regs->trg_addr); |
376 | tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, val: ch_regs->fifo_ctrl); |
377 | tdma_ch_write(tdc, ADMA_CH_CONFIG, val: ch_regs->config); |
378 | |
379 | /* Start ADMA */ |
380 | tdma_ch_write(tdc, ADMA_CH_CMD, val: 1); |
381 | |
382 | tdc->desc = desc; |
383 | } |
384 | |
385 | static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc) |
386 | { |
387 | struct tegra_adma_desc *desc = tdc->desc; |
388 | unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1; |
389 | unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS); |
390 | unsigned int periods_remaining; |
391 | |
392 | /* |
393 | * Handle wrap around of buffer count register |
394 | */ |
395 | if (pos < tdc->tx_buf_pos) |
396 | tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos); |
397 | else |
398 | tdc->tx_buf_count += pos - tdc->tx_buf_pos; |
399 | |
400 | periods_remaining = tdc->tx_buf_count % desc->num_periods; |
401 | tdc->tx_buf_pos = pos; |
402 | |
403 | return desc->buf_len - (periods_remaining * desc->period_len); |
404 | } |
405 | |
406 | static irqreturn_t tegra_adma_isr(int irq, void *dev_id) |
407 | { |
408 | struct tegra_adma_chan *tdc = dev_id; |
409 | unsigned long status; |
410 | |
411 | spin_lock(lock: &tdc->vc.lock); |
412 | |
413 | status = tegra_adma_irq_clear(tdc); |
414 | if (status == 0 || !tdc->desc) { |
415 | spin_unlock(lock: &tdc->vc.lock); |
416 | return IRQ_NONE; |
417 | } |
418 | |
419 | vchan_cyclic_callback(vd: &tdc->desc->vd); |
420 | |
421 | spin_unlock(lock: &tdc->vc.lock); |
422 | |
423 | return IRQ_HANDLED; |
424 | } |
425 | |
426 | static void tegra_adma_issue_pending(struct dma_chan *dc) |
427 | { |
428 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
429 | unsigned long flags; |
430 | |
431 | spin_lock_irqsave(&tdc->vc.lock, flags); |
432 | |
433 | if (vchan_issue_pending(vc: &tdc->vc)) { |
434 | if (!tdc->desc) |
435 | tegra_adma_start(tdc); |
436 | } |
437 | |
438 | spin_unlock_irqrestore(lock: &tdc->vc.lock, flags); |
439 | } |
440 | |
441 | static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc) |
442 | { |
443 | u32 csts; |
444 | |
445 | csts = tdma_ch_read(tdc, ADMA_CH_STATUS); |
446 | csts &= ADMA_CH_STATUS_XFER_PAUSED; |
447 | |
448 | return csts ? true : false; |
449 | } |
450 | |
451 | static int tegra_adma_pause(struct dma_chan *dc) |
452 | { |
453 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
454 | struct tegra_adma_desc *desc = tdc->desc; |
455 | struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; |
456 | int dcnt = 10; |
457 | |
458 | ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); |
459 | ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); |
460 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_regs->ctrl); |
461 | |
462 | while (dcnt-- && !tegra_adma_is_paused(tdc)) |
463 | udelay(TEGRA_ADMA_BURST_COMPLETE_TIME); |
464 | |
465 | if (dcnt < 0) { |
466 | dev_err(tdc2dev(tdc), "unable to pause DMA channel\n" ); |
467 | return -EBUSY; |
468 | } |
469 | |
470 | return 0; |
471 | } |
472 | |
473 | static int tegra_adma_resume(struct dma_chan *dc) |
474 | { |
475 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
476 | struct tegra_adma_desc *desc = tdc->desc; |
477 | struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; |
478 | |
479 | ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); |
480 | ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); |
481 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_regs->ctrl); |
482 | |
483 | return 0; |
484 | } |
485 | |
486 | static int tegra_adma_terminate_all(struct dma_chan *dc) |
487 | { |
488 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
489 | unsigned long flags; |
490 | LIST_HEAD(head); |
491 | |
492 | spin_lock_irqsave(&tdc->vc.lock, flags); |
493 | |
494 | if (tdc->desc) |
495 | tegra_adma_stop(tdc); |
496 | |
497 | tegra_adma_request_free(tdc); |
498 | vchan_get_all_descriptors(vc: &tdc->vc, head: &head); |
499 | spin_unlock_irqrestore(lock: &tdc->vc.lock, flags); |
500 | vchan_dma_desc_free_list(vc: &tdc->vc, head: &head); |
501 | |
502 | return 0; |
503 | } |
504 | |
505 | static enum dma_status tegra_adma_tx_status(struct dma_chan *dc, |
506 | dma_cookie_t cookie, |
507 | struct dma_tx_state *txstate) |
508 | { |
509 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
510 | struct tegra_adma_desc *desc; |
511 | struct virt_dma_desc *vd; |
512 | enum dma_status ret; |
513 | unsigned long flags; |
514 | unsigned int residual; |
515 | |
516 | ret = dma_cookie_status(chan: dc, cookie, state: txstate); |
517 | if (ret == DMA_COMPLETE || !txstate) |
518 | return ret; |
519 | |
520 | spin_lock_irqsave(&tdc->vc.lock, flags); |
521 | |
522 | vd = vchan_find_desc(&tdc->vc, cookie); |
523 | if (vd) { |
524 | desc = to_tegra_adma_desc(td: &vd->tx); |
525 | residual = desc->ch_regs.tc; |
526 | } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) { |
527 | residual = tegra_adma_get_residue(tdc); |
528 | } else { |
529 | residual = 0; |
530 | } |
531 | |
532 | spin_unlock_irqrestore(lock: &tdc->vc.lock, flags); |
533 | |
534 | dma_set_residue(state: txstate, residue: residual); |
535 | |
536 | return ret; |
537 | } |
538 | |
539 | static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size) |
540 | { |
541 | if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) |
542 | burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; |
543 | |
544 | return fls(x: burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; |
545 | } |
546 | |
547 | static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size) |
548 | { |
549 | if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) |
550 | burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; |
551 | |
552 | return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; |
553 | } |
554 | |
555 | static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, |
556 | struct tegra_adma_desc *desc, |
557 | dma_addr_t buf_addr, |
558 | enum dma_transfer_direction direction) |
559 | { |
560 | struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; |
561 | const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata; |
562 | unsigned int burst_size, adma_dir, fifo_size_shift; |
563 | |
564 | if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) |
565 | return -EINVAL; |
566 | |
567 | switch (direction) { |
568 | case DMA_MEM_TO_DEV: |
569 | fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT; |
570 | adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB; |
571 | burst_size = tdc->sconfig.dst_maxburst; |
572 | ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1); |
573 | ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, |
574 | cdata->ch_req_mask, |
575 | cdata->ch_req_tx_shift); |
576 | ch_regs->src_addr = buf_addr; |
577 | break; |
578 | |
579 | case DMA_DEV_TO_MEM: |
580 | fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT; |
581 | adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM; |
582 | burst_size = tdc->sconfig.src_maxburst; |
583 | ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1); |
584 | ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, |
585 | cdata->ch_req_mask, |
586 | cdata->ch_req_rx_shift); |
587 | ch_regs->trg_addr = buf_addr; |
588 | break; |
589 | |
590 | default: |
591 | dev_err(tdc2dev(tdc), "DMA direction is not supported\n" ); |
592 | return -EINVAL; |
593 | } |
594 | |
595 | ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) | |
596 | ADMA_CH_CTRL_MODE_CONTINUOUS | |
597 | ADMA_CH_CTRL_FLOWCTRL_EN; |
598 | ch_regs->config |= cdata->adma_get_burst_config(burst_size); |
599 | ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1); |
600 | if (cdata->has_outstanding_reqs) |
601 | ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8); |
602 | |
603 | /* |
604 | * 'sreq_index' represents the current ADMAIF channel number and as per |
605 | * HW recommendation its FIFO size should match with the corresponding |
606 | * ADMA channel. |
607 | * |
608 | * ADMA FIFO size is set as per below (based on default ADMAIF channel |
609 | * FIFO sizes): |
610 | * fifo_size = 0x2 (sreq_index > sreq_index_offset) |
611 | * fifo_size = 0x3 (sreq_index <= sreq_index_offset) |
612 | * |
613 | */ |
614 | if (tdc->sreq_index > cdata->sreq_index_offset) |
615 | ch_regs->fifo_ctrl = |
616 | ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask, |
617 | fifo_size_shift); |
618 | else |
619 | ch_regs->fifo_ctrl = |
620 | ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask, |
621 | fifo_size_shift); |
622 | |
623 | ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK; |
624 | |
625 | return tegra_adma_request_alloc(tdc, direction); |
626 | } |
627 | |
628 | static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic( |
629 | struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, |
630 | size_t period_len, enum dma_transfer_direction direction, |
631 | unsigned long flags) |
632 | { |
633 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
634 | struct tegra_adma_desc *desc = NULL; |
635 | |
636 | if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) { |
637 | dev_err(tdc2dev(tdc), "invalid buffer/period len\n" ); |
638 | return NULL; |
639 | } |
640 | |
641 | if (buf_len % period_len) { |
642 | dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n" ); |
643 | return NULL; |
644 | } |
645 | |
646 | if (!IS_ALIGNED(buf_addr, 4)) { |
647 | dev_err(tdc2dev(tdc), "invalid buffer alignment\n" ); |
648 | return NULL; |
649 | } |
650 | |
651 | desc = kzalloc(size: sizeof(*desc), GFP_NOWAIT); |
652 | if (!desc) |
653 | return NULL; |
654 | |
655 | desc->buf_len = buf_len; |
656 | desc->period_len = period_len; |
657 | desc->num_periods = buf_len / period_len; |
658 | |
659 | if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) { |
660 | kfree(objp: desc); |
661 | return NULL; |
662 | } |
663 | |
664 | return vchan_tx_prep(vc: &tdc->vc, vd: &desc->vd, tx_flags: flags); |
665 | } |
666 | |
667 | static int tegra_adma_alloc_chan_resources(struct dma_chan *dc) |
668 | { |
669 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
670 | int ret; |
671 | |
672 | ret = request_irq(irq: tdc->irq, handler: tegra_adma_isr, flags: 0, name: dma_chan_name(chan: dc), dev: tdc); |
673 | if (ret) { |
674 | dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n" , |
675 | dma_chan_name(dc)); |
676 | return ret; |
677 | } |
678 | |
679 | ret = pm_runtime_resume_and_get(dev: tdc2dev(tdc)); |
680 | if (ret < 0) { |
681 | free_irq(tdc->irq, tdc); |
682 | return ret; |
683 | } |
684 | |
685 | dma_cookie_init(chan: &tdc->vc.chan); |
686 | |
687 | return 0; |
688 | } |
689 | |
690 | static void tegra_adma_free_chan_resources(struct dma_chan *dc) |
691 | { |
692 | struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); |
693 | |
694 | tegra_adma_terminate_all(dc); |
695 | vchan_free_chan_resources(vc: &tdc->vc); |
696 | tasklet_kill(t: &tdc->vc.task); |
697 | free_irq(tdc->irq, tdc); |
698 | pm_runtime_put(dev: tdc2dev(tdc)); |
699 | |
700 | tdc->sreq_index = 0; |
701 | tdc->sreq_dir = DMA_TRANS_NONE; |
702 | } |
703 | |
704 | static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, |
705 | struct of_dma *ofdma) |
706 | { |
707 | struct tegra_adma *tdma = ofdma->of_dma_data; |
708 | struct tegra_adma_chan *tdc; |
709 | struct dma_chan *chan; |
710 | unsigned int sreq_index; |
711 | |
712 | if (dma_spec->args_count != 1) |
713 | return NULL; |
714 | |
715 | sreq_index = dma_spec->args[0]; |
716 | |
717 | if (sreq_index == 0) { |
718 | dev_err(tdma->dev, "DMA request must not be 0\n" ); |
719 | return NULL; |
720 | } |
721 | |
722 | chan = dma_get_any_slave_channel(device: &tdma->dma_dev); |
723 | if (!chan) |
724 | return NULL; |
725 | |
726 | tdc = to_tegra_adma_chan(dc: chan); |
727 | tdc->sreq_index = sreq_index; |
728 | |
729 | return chan; |
730 | } |
731 | |
732 | static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) |
733 | { |
734 | struct tegra_adma *tdma = dev_get_drvdata(dev); |
735 | struct tegra_adma_chan_regs *ch_reg; |
736 | struct tegra_adma_chan *tdc; |
737 | int i; |
738 | |
739 | tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); |
740 | if (!tdma->global_cmd) |
741 | goto clk_disable; |
742 | |
743 | for (i = 0; i < tdma->nr_channels; i++) { |
744 | tdc = &tdma->channels[i]; |
745 | /* skip for reserved channels */ |
746 | if (!tdc->tdma) |
747 | continue; |
748 | |
749 | ch_reg = &tdc->ch_regs; |
750 | ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); |
751 | /* skip if channel is not active */ |
752 | if (!ch_reg->cmd) |
753 | continue; |
754 | ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC); |
755 | ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR); |
756 | ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR); |
757 | ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); |
758 | ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); |
759 | ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG); |
760 | } |
761 | |
762 | clk_disable: |
763 | clk_disable_unprepare(clk: tdma->ahub_clk); |
764 | |
765 | return 0; |
766 | } |
767 | |
768 | static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) |
769 | { |
770 | struct tegra_adma *tdma = dev_get_drvdata(dev); |
771 | struct tegra_adma_chan_regs *ch_reg; |
772 | struct tegra_adma_chan *tdc; |
773 | int ret, i; |
774 | |
775 | ret = clk_prepare_enable(clk: tdma->ahub_clk); |
776 | if (ret) { |
777 | dev_err(dev, "ahub clk_enable failed: %d\n" , ret); |
778 | return ret; |
779 | } |
780 | tdma_write(tdma, ADMA_GLOBAL_CMD, val: tdma->global_cmd); |
781 | |
782 | if (!tdma->global_cmd) |
783 | return 0; |
784 | |
785 | for (i = 0; i < tdma->nr_channels; i++) { |
786 | tdc = &tdma->channels[i]; |
787 | /* skip for reserved channels */ |
788 | if (!tdc->tdma) |
789 | continue; |
790 | ch_reg = &tdc->ch_regs; |
791 | /* skip if channel was not active earlier */ |
792 | if (!ch_reg->cmd) |
793 | continue; |
794 | tdma_ch_write(tdc, ADMA_CH_TC, val: ch_reg->tc); |
795 | tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, val: ch_reg->src_addr); |
796 | tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, val: ch_reg->trg_addr); |
797 | tdma_ch_write(tdc, ADMA_CH_CTRL, val: ch_reg->ctrl); |
798 | tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, val: ch_reg->fifo_ctrl); |
799 | tdma_ch_write(tdc, ADMA_CH_CONFIG, val: ch_reg->config); |
800 | tdma_ch_write(tdc, ADMA_CH_CMD, val: ch_reg->cmd); |
801 | } |
802 | |
803 | return 0; |
804 | } |
805 | |
806 | static const struct tegra_adma_chip_data tegra210_chip_data = { |
807 | .adma_get_burst_config = tegra210_adma_get_burst_config, |
808 | .global_reg_offset = 0xc00, |
809 | .global_int_clear = 0x20, |
810 | .ch_req_tx_shift = 28, |
811 | .ch_req_rx_shift = 24, |
812 | .ch_base_offset = 0, |
813 | .ch_req_mask = 0xf, |
814 | .ch_req_max = 10, |
815 | .ch_reg_size = 0x80, |
816 | .nr_channels = 22, |
817 | .ch_fifo_size_mask = 0xf, |
818 | .sreq_index_offset = 2, |
819 | .has_outstanding_reqs = false, |
820 | }; |
821 | |
822 | static const struct tegra_adma_chip_data tegra186_chip_data = { |
823 | .adma_get_burst_config = tegra186_adma_get_burst_config, |
824 | .global_reg_offset = 0, |
825 | .global_int_clear = 0x402c, |
826 | .ch_req_tx_shift = 27, |
827 | .ch_req_rx_shift = 22, |
828 | .ch_base_offset = 0x10000, |
829 | .ch_req_mask = 0x1f, |
830 | .ch_req_max = 20, |
831 | .ch_reg_size = 0x100, |
832 | .nr_channels = 32, |
833 | .ch_fifo_size_mask = 0x1f, |
834 | .sreq_index_offset = 4, |
835 | .has_outstanding_reqs = true, |
836 | }; |
837 | |
838 | static const struct of_device_id tegra_adma_of_match[] = { |
839 | { .compatible = "nvidia,tegra210-adma" , .data = &tegra210_chip_data }, |
840 | { .compatible = "nvidia,tegra186-adma" , .data = &tegra186_chip_data }, |
841 | { }, |
842 | }; |
843 | MODULE_DEVICE_TABLE(of, tegra_adma_of_match); |
844 | |
845 | static int tegra_adma_probe(struct platform_device *pdev) |
846 | { |
847 | const struct tegra_adma_chip_data *cdata; |
848 | struct tegra_adma *tdma; |
849 | int ret, i; |
850 | |
851 | cdata = of_device_get_match_data(dev: &pdev->dev); |
852 | if (!cdata) { |
853 | dev_err(&pdev->dev, "device match data not found\n" ); |
854 | return -ENODEV; |
855 | } |
856 | |
857 | tdma = devm_kzalloc(dev: &pdev->dev, |
858 | struct_size(tdma, channels, cdata->nr_channels), |
859 | GFP_KERNEL); |
860 | if (!tdma) |
861 | return -ENOMEM; |
862 | |
863 | tdma->dev = &pdev->dev; |
864 | tdma->cdata = cdata; |
865 | tdma->nr_channels = cdata->nr_channels; |
866 | platform_set_drvdata(pdev, data: tdma); |
867 | |
868 | tdma->base_addr = devm_platform_ioremap_resource(pdev, index: 0); |
869 | if (IS_ERR(ptr: tdma->base_addr)) |
870 | return PTR_ERR(ptr: tdma->base_addr); |
871 | |
872 | tdma->ahub_clk = devm_clk_get(dev: &pdev->dev, id: "d_audio" ); |
873 | if (IS_ERR(ptr: tdma->ahub_clk)) { |
874 | dev_err(&pdev->dev, "Error: Missing ahub controller clock\n" ); |
875 | return PTR_ERR(ptr: tdma->ahub_clk); |
876 | } |
877 | |
878 | tdma->dma_chan_mask = devm_kzalloc(dev: &pdev->dev, |
879 | BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long), |
880 | GFP_KERNEL); |
881 | if (!tdma->dma_chan_mask) |
882 | return -ENOMEM; |
883 | |
884 | /* Enable all channels by default */ |
885 | bitmap_fill(dst: tdma->dma_chan_mask, nbits: tdma->nr_channels); |
886 | |
887 | ret = of_property_read_u32_array(np: pdev->dev.of_node, propname: "dma-channel-mask" , |
888 | out_values: (u32 *)tdma->dma_chan_mask, |
889 | BITS_TO_U32(tdma->nr_channels)); |
890 | if (ret < 0 && (ret != -EINVAL)) { |
891 | dev_err(&pdev->dev, "dma-channel-mask is not complete.\n" ); |
892 | return ret; |
893 | } |
894 | |
895 | INIT_LIST_HEAD(list: &tdma->dma_dev.channels); |
896 | for (i = 0; i < tdma->nr_channels; i++) { |
897 | struct tegra_adma_chan *tdc = &tdma->channels[i]; |
898 | |
899 | /* skip for reserved channels */ |
900 | if (!test_bit(i, tdma->dma_chan_mask)) |
901 | continue; |
902 | |
903 | tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset |
904 | + (cdata->ch_reg_size * i); |
905 | |
906 | tdc->irq = of_irq_get(dev: pdev->dev.of_node, index: i); |
907 | if (tdc->irq <= 0) { |
908 | ret = tdc->irq ?: -ENXIO; |
909 | goto irq_dispose; |
910 | } |
911 | |
912 | vchan_init(vc: &tdc->vc, dmadev: &tdma->dma_dev); |
913 | tdc->vc.desc_free = tegra_adma_desc_free; |
914 | tdc->tdma = tdma; |
915 | } |
916 | |
917 | pm_runtime_enable(dev: &pdev->dev); |
918 | |
919 | ret = pm_runtime_resume_and_get(dev: &pdev->dev); |
920 | if (ret < 0) |
921 | goto rpm_disable; |
922 | |
923 | ret = tegra_adma_init(tdma); |
924 | if (ret) |
925 | goto rpm_put; |
926 | |
927 | dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); |
928 | dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); |
929 | dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); |
930 | |
931 | tdma->dma_dev.dev = &pdev->dev; |
932 | tdma->dma_dev.device_alloc_chan_resources = |
933 | tegra_adma_alloc_chan_resources; |
934 | tdma->dma_dev.device_free_chan_resources = |
935 | tegra_adma_free_chan_resources; |
936 | tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending; |
937 | tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic; |
938 | tdma->dma_dev.device_config = tegra_adma_slave_config; |
939 | tdma->dma_dev.device_tx_status = tegra_adma_tx_status; |
940 | tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all; |
941 | tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
942 | tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); |
943 | tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); |
944 | tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; |
945 | tdma->dma_dev.device_pause = tegra_adma_pause; |
946 | tdma->dma_dev.device_resume = tegra_adma_resume; |
947 | |
948 | ret = dma_async_device_register(device: &tdma->dma_dev); |
949 | if (ret < 0) { |
950 | dev_err(&pdev->dev, "ADMA registration failed: %d\n" , ret); |
951 | goto rpm_put; |
952 | } |
953 | |
954 | ret = of_dma_controller_register(np: pdev->dev.of_node, |
955 | of_dma_xlate: tegra_dma_of_xlate, data: tdma); |
956 | if (ret < 0) { |
957 | dev_err(&pdev->dev, "ADMA OF registration failed %d\n" , ret); |
958 | goto dma_remove; |
959 | } |
960 | |
961 | pm_runtime_put(dev: &pdev->dev); |
962 | |
963 | dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n" , |
964 | tdma->nr_channels); |
965 | |
966 | return 0; |
967 | |
968 | dma_remove: |
969 | dma_async_device_unregister(device: &tdma->dma_dev); |
970 | rpm_put: |
971 | pm_runtime_put_sync(dev: &pdev->dev); |
972 | rpm_disable: |
973 | pm_runtime_disable(dev: &pdev->dev); |
974 | irq_dispose: |
975 | while (--i >= 0) |
976 | irq_dispose_mapping(virq: tdma->channels[i].irq); |
977 | |
978 | return ret; |
979 | } |
980 | |
981 | static void tegra_adma_remove(struct platform_device *pdev) |
982 | { |
983 | struct tegra_adma *tdma = platform_get_drvdata(pdev); |
984 | int i; |
985 | |
986 | of_dma_controller_free(np: pdev->dev.of_node); |
987 | dma_async_device_unregister(device: &tdma->dma_dev); |
988 | |
989 | for (i = 0; i < tdma->nr_channels; ++i) { |
990 | if (tdma->channels[i].irq) |
991 | irq_dispose_mapping(virq: tdma->channels[i].irq); |
992 | } |
993 | |
994 | pm_runtime_disable(dev: &pdev->dev); |
995 | } |
996 | |
997 | static const struct dev_pm_ops tegra_adma_dev_pm_ops = { |
998 | SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend, |
999 | tegra_adma_runtime_resume, NULL) |
1000 | SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, |
1001 | pm_runtime_force_resume) |
1002 | }; |
1003 | |
1004 | static struct platform_driver tegra_admac_driver = { |
1005 | .driver = { |
1006 | .name = "tegra-adma" , |
1007 | .pm = &tegra_adma_dev_pm_ops, |
1008 | .of_match_table = tegra_adma_of_match, |
1009 | }, |
1010 | .probe = tegra_adma_probe, |
1011 | .remove_new = tegra_adma_remove, |
1012 | }; |
1013 | |
1014 | module_platform_driver(tegra_admac_driver); |
1015 | |
1016 | MODULE_ALIAS("platform:tegra210-adma" ); |
1017 | MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver" ); |
1018 | MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>" ); |
1019 | MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>" ); |
1020 | MODULE_LICENSE("GPL v2" ); |
1021 | |