1// SPDX-License-Identifier: GPL-2.0
2// Copyright (C) 2024-2025 Arm Limited
3// Arm DMA-350 driver
4
5#include <linux/bitfield.h>
6#include <linux/dmaengine.h>
7#include <linux/dma-mapping.h>
8#include <linux/io.h>
9#include <linux/of.h>
10#include <linux/module.h>
11#include <linux/platform_device.h>
12
13#include "dmaengine.h"
14#include "virt-dma.h"
15
16#define DMAINFO 0x0f00
17
18#define DMA_BUILDCFG0 0xb0
19#define DMA_CFG_DATA_WIDTH GENMASK(18, 16)
20#define DMA_CFG_ADDR_WIDTH GENMASK(15, 10)
21#define DMA_CFG_NUM_CHANNELS GENMASK(9, 4)
22
23#define DMA_BUILDCFG1 0xb4
24#define DMA_CFG_NUM_TRIGGER_IN GENMASK(8, 0)
25
26#define IIDR 0xc8
27#define IIDR_PRODUCTID GENMASK(31, 20)
28#define IIDR_VARIANT GENMASK(19, 16)
29#define IIDR_REVISION GENMASK(15, 12)
30#define IIDR_IMPLEMENTER GENMASK(11, 0)
31
32#define PRODUCTID_DMA350 0x3a0
33#define IMPLEMENTER_ARM 0x43b
34
35#define DMACH(n) (0x1000 + 0x0100 * (n))
36
37#define CH_CMD 0x00
38#define CH_CMD_RESUME BIT(5)
39#define CH_CMD_PAUSE BIT(4)
40#define CH_CMD_STOP BIT(3)
41#define CH_CMD_DISABLE BIT(2)
42#define CH_CMD_CLEAR BIT(1)
43#define CH_CMD_ENABLE BIT(0)
44
45#define CH_STATUS 0x04
46#define CH_STAT_RESUMEWAIT BIT(21)
47#define CH_STAT_PAUSED BIT(20)
48#define CH_STAT_STOPPED BIT(19)
49#define CH_STAT_DISABLED BIT(18)
50#define CH_STAT_ERR BIT(17)
51#define CH_STAT_DONE BIT(16)
52#define CH_STAT_INTR_ERR BIT(1)
53#define CH_STAT_INTR_DONE BIT(0)
54
55#define CH_INTREN 0x08
56#define CH_INTREN_ERR BIT(1)
57#define CH_INTREN_DONE BIT(0)
58
59#define CH_CTRL 0x0c
60#define CH_CTRL_USEDESTRIGIN BIT(26)
61#define CH_CTRL_USESRCTRIGIN BIT(26)
62#define CH_CTRL_DONETYPE GENMASK(23, 21)
63#define CH_CTRL_REGRELOADTYPE GENMASK(20, 18)
64#define CH_CTRL_XTYPE GENMASK(11, 9)
65#define CH_CTRL_TRANSIZE GENMASK(2, 0)
66
67#define CH_SRCADDR 0x10
68#define CH_SRCADDRHI 0x14
69#define CH_DESADDR 0x18
70#define CH_DESADDRHI 0x1c
71#define CH_XSIZE 0x20
72#define CH_XSIZEHI 0x24
73#define CH_SRCTRANSCFG 0x28
74#define CH_DESTRANSCFG 0x2c
75#define CH_CFG_MAXBURSTLEN GENMASK(19, 16)
76#define CH_CFG_PRIVATTR BIT(11)
77#define CH_CFG_SHAREATTR GENMASK(9, 8)
78#define CH_CFG_MEMATTR GENMASK(7, 0)
79
80#define TRANSCFG_DEVICE \
81 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \
82 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \
83 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE)
84#define TRANSCFG_NC \
85 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \
86 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \
87 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC)
88#define TRANSCFG_WB \
89 FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \
90 FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) | \
91 FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB)
92
93#define CH_XADDRINC 0x30
94#define CH_XY_DES GENMASK(31, 16)
95#define CH_XY_SRC GENMASK(15, 0)
96
97#define CH_FILLVAL 0x38
98#define CH_SRCTRIGINCFG 0x4c
99#define CH_DESTRIGINCFG 0x50
100#define CH_LINKATTR 0x70
101#define CH_LINK_SHAREATTR GENMASK(9, 8)
102#define CH_LINK_MEMATTR GENMASK(7, 0)
103
104#define CH_AUTOCFG 0x74
105#define CH_LINKADDR 0x78
106#define CH_LINKADDR_EN BIT(0)
107
108#define CH_LINKADDRHI 0x7c
109#define CH_ERRINFO 0x90
110#define CH_ERRINFO_AXIRDPOISERR BIT(18)
111#define CH_ERRINFO_AXIWRRESPERR BIT(17)
112#define CH_ERRINFO_AXIRDRESPERR BIT(16)
113
114#define CH_BUILDCFG0 0xf8
115#define CH_CFG_INC_WIDTH GENMASK(29, 26)
116#define CH_CFG_DATA_WIDTH GENMASK(24, 22)
117#define CH_CFG_DATA_BUF_SIZE GENMASK(7, 0)
118
119#define CH_BUILDCFG1 0xfc
120#define CH_CFG_HAS_CMDLINK BIT(8)
121#define CH_CFG_HAS_TRIGSEL BIT(7)
122#define CH_CFG_HAS_TRIGIN BIT(5)
123#define CH_CFG_HAS_WRAP BIT(1)
124
125
126#define LINK_REGCLEAR BIT(0)
127#define LINK_INTREN BIT(2)
128#define LINK_CTRL BIT(3)
129#define LINK_SRCADDR BIT(4)
130#define LINK_SRCADDRHI BIT(5)
131#define LINK_DESADDR BIT(6)
132#define LINK_DESADDRHI BIT(7)
133#define LINK_XSIZE BIT(8)
134#define LINK_XSIZEHI BIT(9)
135#define LINK_SRCTRANSCFG BIT(10)
136#define LINK_DESTRANSCFG BIT(11)
137#define LINK_XADDRINC BIT(12)
138#define LINK_FILLVAL BIT(14)
139#define LINK_SRCTRIGINCFG BIT(19)
140#define LINK_DESTRIGINCFG BIT(20)
141#define LINK_AUTOCFG BIT(29)
142#define LINK_LINKADDR BIT(30)
143#define LINK_LINKADDRHI BIT(31)
144
145
146enum ch_ctrl_donetype {
147 CH_CTRL_DONETYPE_NONE = 0,
148 CH_CTRL_DONETYPE_CMD = 1,
149 CH_CTRL_DONETYPE_CYCLE = 3
150};
151
152enum ch_ctrl_xtype {
153 CH_CTRL_XTYPE_DISABLE = 0,
154 CH_CTRL_XTYPE_CONTINUE = 1,
155 CH_CTRL_XTYPE_WRAP = 2,
156 CH_CTRL_XTYPE_FILL = 3
157};
158
159enum ch_cfg_shareattr {
160 SHAREATTR_NSH = 0,
161 SHAREATTR_OSH = 2,
162 SHAREATTR_ISH = 3
163};
164
165enum ch_cfg_memattr {
166 MEMATTR_DEVICE = 0x00,
167 MEMATTR_NC = 0x44,
168 MEMATTR_WB = 0xff
169};
170
171struct d350_desc {
172 struct virt_dma_desc vd;
173 u32 command[16];
174 u16 xsize;
175 u16 xsizehi;
176 u8 tsz;
177};
178
179struct d350_chan {
180 struct virt_dma_chan vc;
181 struct d350_desc *desc;
182 void __iomem *base;
183 int irq;
184 enum dma_status status;
185 dma_cookie_t cookie;
186 u32 residue;
187 u8 tsz;
188 bool has_trig;
189 bool has_wrap;
190 bool coherent;
191};
192
193struct d350 {
194 struct dma_device dma;
195 int nchan;
196 int nreq;
197 struct d350_chan channels[] __counted_by(nchan);
198};
199
200static inline struct d350_chan *to_d350_chan(struct dma_chan *chan)
201{
202 return container_of(chan, struct d350_chan, vc.chan);
203}
204
205static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd)
206{
207 return container_of(vd, struct d350_desc, vd);
208}
209
210static void d350_desc_free(struct virt_dma_desc *vd)
211{
212 kfree(objp: to_d350_desc(vd));
213}
214
215static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan,
216 dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags)
217{
218 struct d350_chan *dch = to_d350_chan(chan);
219 struct d350_desc *desc;
220 u32 *cmd;
221
222 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
223 if (!desc)
224 return NULL;
225
226 desc->tsz = __ffs(len | dest | src | (1 << dch->tsz));
227 desc->xsize = lower_16_bits(len >> desc->tsz);
228 desc->xsizehi = upper_16_bits(len >> desc->tsz);
229
230 cmd = desc->command;
231 cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR |
232 LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG |
233 LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR;
234
235 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
236 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) |
237 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD);
238
239 cmd[2] = lower_32_bits(src);
240 cmd[3] = upper_32_bits(src);
241 cmd[4] = lower_32_bits(dest);
242 cmd[5] = upper_32_bits(dest);
243 cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize);
244 cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi);
245 cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
246 cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
247 cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1);
248 cmd[11] = 0;
249
250 return vchan_tx_prep(vc: &dch->vc, vd: &desc->vd, tx_flags: flags);
251}
252
253static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan,
254 dma_addr_t dest, int value, size_t len, unsigned long flags)
255{
256 struct d350_chan *dch = to_d350_chan(chan);
257 struct d350_desc *desc;
258 u32 *cmd;
259
260 desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
261 if (!desc)
262 return NULL;
263
264 desc->tsz = __ffs(len | dest | (1 << dch->tsz));
265 desc->xsize = lower_16_bits(len >> desc->tsz);
266 desc->xsizehi = upper_16_bits(len >> desc->tsz);
267
268 cmd = desc->command;
269 cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI |
270 LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG |
271 LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR;
272
273 cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
274 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) |
275 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD);
276
277 cmd[2] = lower_32_bits(dest);
278 cmd[3] = upper_32_bits(dest);
279 cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize);
280 cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi);
281 cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
282 cmd[7] = FIELD_PREP(CH_XY_DES, 1);
283 cmd[8] = (u8)value * 0x01010101;
284 cmd[9] = 0;
285
286 return vchan_tx_prep(vc: &dch->vc, vd: &desc->vd, tx_flags: flags);
287}
288
289static int d350_pause(struct dma_chan *chan)
290{
291 struct d350_chan *dch = to_d350_chan(chan);
292 unsigned long flags;
293
294 spin_lock_irqsave(&dch->vc.lock, flags);
295 if (dch->status == DMA_IN_PROGRESS) {
296 writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD);
297 dch->status = DMA_PAUSED;
298 }
299 spin_unlock_irqrestore(lock: &dch->vc.lock, flags);
300
301 return 0;
302}
303
304static int d350_resume(struct dma_chan *chan)
305{
306 struct d350_chan *dch = to_d350_chan(chan);
307 unsigned long flags;
308
309 spin_lock_irqsave(&dch->vc.lock, flags);
310 if (dch->status == DMA_PAUSED) {
311 writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD);
312 dch->status = DMA_IN_PROGRESS;
313 }
314 spin_unlock_irqrestore(lock: &dch->vc.lock, flags);
315
316 return 0;
317}
318
319static u32 d350_get_residue(struct d350_chan *dch)
320{
321 u32 res, xsize, xsizehi, hi_new;
322 int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
323
324 hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
325 do {
326 xsizehi = hi_new;
327 xsize = readl_relaxed(dch->base + CH_XSIZE);
328 hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
329 } while (xsizehi != hi_new && --retries);
330
331 res = FIELD_GET(CH_XY_DES, xsize);
332 res |= FIELD_GET(CH_XY_DES, xsizehi) << 16;
333
334 return res << dch->desc->tsz;
335}
336
337static int d350_terminate_all(struct dma_chan *chan)
338{
339 struct d350_chan *dch = to_d350_chan(chan);
340 unsigned long flags;
341 LIST_HEAD(list);
342
343 spin_lock_irqsave(&dch->vc.lock, flags);
344 writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD);
345 if (dch->desc) {
346 if (dch->status != DMA_ERROR)
347 vchan_terminate_vdesc(vd: &dch->desc->vd);
348 dch->desc = NULL;
349 dch->status = DMA_COMPLETE;
350 }
351 vchan_get_all_descriptors(vc: &dch->vc, head: &list);
352 list_splice_tail(list: &list, head: &dch->vc.desc_terminated);
353 spin_unlock_irqrestore(lock: &dch->vc.lock, flags);
354
355 return 0;
356}
357
358static void d350_synchronize(struct dma_chan *chan)
359{
360 struct d350_chan *dch = to_d350_chan(chan);
361
362 vchan_synchronize(vc: &dch->vc);
363}
364
365static u32 d350_desc_bytes(struct d350_desc *desc)
366{
367 return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz;
368}
369
370static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
371 struct dma_tx_state *state)
372{
373 struct d350_chan *dch = to_d350_chan(chan);
374 struct virt_dma_desc *vd;
375 enum dma_status status;
376 unsigned long flags;
377 u32 residue = 0;
378
379 status = dma_cookie_status(chan, cookie, state);
380
381 spin_lock_irqsave(&dch->vc.lock, flags);
382 if (cookie == dch->cookie) {
383 status = dch->status;
384 if (status == DMA_IN_PROGRESS || status == DMA_PAUSED)
385 dch->residue = d350_get_residue(dch);
386 residue = dch->residue;
387 } else if ((vd = vchan_find_desc(&dch->vc, cookie))) {
388 residue = d350_desc_bytes(desc: to_d350_desc(vd));
389 } else if (status == DMA_IN_PROGRESS) {
390 /* Somebody else terminated it? */
391 status = DMA_ERROR;
392 }
393 spin_unlock_irqrestore(lock: &dch->vc.lock, flags);
394
395 dma_set_residue(state, residue);
396 return status;
397}
398
399static void d350_start_next(struct d350_chan *dch)
400{
401 u32 hdr, *reg;
402
403 dch->desc = to_d350_desc(vd: vchan_next_desc(vc: &dch->vc));
404 if (!dch->desc)
405 return;
406
407 list_del(entry: &dch->desc->vd.node);
408 dch->status = DMA_IN_PROGRESS;
409 dch->cookie = dch->desc->vd.tx.cookie;
410 dch->residue = d350_desc_bytes(desc: dch->desc);
411
412 hdr = dch->desc->command[0];
413 reg = &dch->desc->command[1];
414
415 if (hdr & LINK_INTREN)
416 writel_relaxed(*reg++, dch->base + CH_INTREN);
417 if (hdr & LINK_CTRL)
418 writel_relaxed(*reg++, dch->base + CH_CTRL);
419 if (hdr & LINK_SRCADDR)
420 writel_relaxed(*reg++, dch->base + CH_SRCADDR);
421 if (hdr & LINK_SRCADDRHI)
422 writel_relaxed(*reg++, dch->base + CH_SRCADDRHI);
423 if (hdr & LINK_DESADDR)
424 writel_relaxed(*reg++, dch->base + CH_DESADDR);
425 if (hdr & LINK_DESADDRHI)
426 writel_relaxed(*reg++, dch->base + CH_DESADDRHI);
427 if (hdr & LINK_XSIZE)
428 writel_relaxed(*reg++, dch->base + CH_XSIZE);
429 if (hdr & LINK_XSIZEHI)
430 writel_relaxed(*reg++, dch->base + CH_XSIZEHI);
431 if (hdr & LINK_SRCTRANSCFG)
432 writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG);
433 if (hdr & LINK_DESTRANSCFG)
434 writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG);
435 if (hdr & LINK_XADDRINC)
436 writel_relaxed(*reg++, dch->base + CH_XADDRINC);
437 if (hdr & LINK_FILLVAL)
438 writel_relaxed(*reg++, dch->base + CH_FILLVAL);
439 if (hdr & LINK_SRCTRIGINCFG)
440 writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG);
441 if (hdr & LINK_DESTRIGINCFG)
442 writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG);
443 if (hdr & LINK_AUTOCFG)
444 writel_relaxed(*reg++, dch->base + CH_AUTOCFG);
445 if (hdr & LINK_LINKADDR)
446 writel_relaxed(*reg++, dch->base + CH_LINKADDR);
447 if (hdr & LINK_LINKADDRHI)
448 writel_relaxed(*reg++, dch->base + CH_LINKADDRHI);
449
450 writel(CH_CMD_ENABLE, addr: dch->base + CH_CMD);
451}
452
453static void d350_issue_pending(struct dma_chan *chan)
454{
455 struct d350_chan *dch = to_d350_chan(chan);
456 unsigned long flags;
457
458 spin_lock_irqsave(&dch->vc.lock, flags);
459 if (vchan_issue_pending(vc: &dch->vc) && !dch->desc)
460 d350_start_next(dch);
461 spin_unlock_irqrestore(lock: &dch->vc.lock, flags);
462}
463
464static irqreturn_t d350_irq(int irq, void *data)
465{
466 struct d350_chan *dch = data;
467 struct device *dev = dch->vc.chan.device->dev;
468 struct virt_dma_desc *vd = &dch->desc->vd;
469 u32 ch_status;
470
471 ch_status = readl(addr: dch->base + CH_STATUS);
472 if (!ch_status)
473 return IRQ_NONE;
474
475 if (ch_status & CH_STAT_INTR_ERR) {
476 u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO);
477
478 if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR))
479 vd->tx_result.result = DMA_TRANS_READ_FAILED;
480 else if (errinfo & CH_ERRINFO_AXIWRRESPERR)
481 vd->tx_result.result = DMA_TRANS_WRITE_FAILED;
482 else
483 vd->tx_result.result = DMA_TRANS_ABORTED;
484
485 vd->tx_result.residue = d350_get_residue(dch);
486 } else if (!(ch_status & CH_STAT_INTR_DONE)) {
487 dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status);
488 }
489 writel_relaxed(ch_status, dch->base + CH_STATUS);
490
491 spin_lock(lock: &dch->vc.lock);
492 vchan_cookie_complete(vd);
493 if (ch_status & CH_STAT_INTR_DONE) {
494 dch->status = DMA_COMPLETE;
495 dch->residue = 0;
496 d350_start_next(dch);
497 } else {
498 dch->status = DMA_ERROR;
499 dch->residue = vd->tx_result.residue;
500 }
501 spin_unlock(lock: &dch->vc.lock);
502
503 return IRQ_HANDLED;
504}
505
506static int d350_alloc_chan_resources(struct dma_chan *chan)
507{
508 struct d350_chan *dch = to_d350_chan(chan);
509 int ret = request_irq(irq: dch->irq, handler: d350_irq, IRQF_SHARED,
510 name: dev_name(dev: &dch->vc.chan.dev->device), dev: dch);
511 if (!ret)
512 writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN);
513
514 return ret;
515}
516
517static void d350_free_chan_resources(struct dma_chan *chan)
518{
519 struct d350_chan *dch = to_d350_chan(chan);
520
521 writel_relaxed(0, dch->base + CH_INTREN);
522 free_irq(dch->irq, dch);
523 vchan_free_chan_resources(vc: &dch->vc);
524}
525
526static int d350_probe(struct platform_device *pdev)
527{
528 struct device *dev = &pdev->dev;
529 struct d350 *dmac;
530 void __iomem *base;
531 u32 reg;
532 int ret, nchan, dw, aw, r, p;
533 bool coherent, memset;
534
535 base = devm_platform_ioremap_resource(pdev, index: 0);
536 if (IS_ERR(ptr: base))
537 return PTR_ERR(ptr: base);
538
539 reg = readl_relaxed(base + DMAINFO + IIDR);
540 r = FIELD_GET(IIDR_VARIANT, reg);
541 p = FIELD_GET(IIDR_REVISION, reg);
542 if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM ||
543 FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350)
544 return dev_err_probe(dev, err: -ENODEV, fmt: "Not a DMA-350!");
545
546 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0);
547 nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1;
548 dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg);
549 aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1;
550
551 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw));
552 coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT;
553
554 dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL);
555 if (!dmac)
556 return -ENOMEM;
557
558 dmac->nchan = nchan;
559
560 reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1);
561 dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg);
562
563 dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq);
564
565 dmac->dma.dev = dev;
566 for (int i = min(dw, 16); i > 0; i /= 2) {
567 dmac->dma.src_addr_widths |= BIT(i);
568 dmac->dma.dst_addr_widths |= BIT(i);
569 }
570 dmac->dma.directions = BIT(DMA_MEM_TO_MEM);
571 dmac->dma.descriptor_reuse = true;
572 dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
573 dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources;
574 dmac->dma.device_free_chan_resources = d350_free_chan_resources;
575 dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask);
576 dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy;
577 dmac->dma.device_pause = d350_pause;
578 dmac->dma.device_resume = d350_resume;
579 dmac->dma.device_terminate_all = d350_terminate_all;
580 dmac->dma.device_synchronize = d350_synchronize;
581 dmac->dma.device_tx_status = d350_tx_status;
582 dmac->dma.device_issue_pending = d350_issue_pending;
583 INIT_LIST_HEAD(list: &dmac->dma.channels);
584
585 /* Would be nice to have per-channel caps for this... */
586 memset = true;
587 for (int i = 0; i < nchan; i++) {
588 struct d350_chan *dch = &dmac->channels[i];
589
590 dch->base = base + DMACH(i);
591 writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD);
592
593 reg = readl_relaxed(dch->base + CH_BUILDCFG1);
594 if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) {
595 dev_warn(dev, "No command link support on channel %d\n", i);
596 continue;
597 }
598 dch->irq = platform_get_irq(pdev, i);
599 if (dch->irq < 0)
600 return dev_err_probe(dev, err: dch->irq,
601 fmt: "Failed to get IRQ for channel %d\n", i);
602
603 dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg);
604 dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) &
605 FIELD_GET(CH_CFG_HAS_TRIGSEL, reg);
606
607 /* Fill is a special case of Wrap */
608 memset &= dch->has_wrap;
609
610 reg = readl_relaxed(dch->base + CH_BUILDCFG0);
611 dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg);
612
613 reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH);
614 reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC);
615 writel_relaxed(reg, dch->base + CH_LINKATTR);
616
617 dch->vc.desc_free = d350_desc_free;
618 vchan_init(vc: &dch->vc, dmadev: &dmac->dma);
619 }
620
621 if (memset) {
622 dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask);
623 dmac->dma.device_prep_dma_memset = d350_prep_memset;
624 }
625
626 platform_set_drvdata(pdev, data: dmac);
627
628 ret = dma_async_device_register(device: &dmac->dma);
629 if (ret)
630 return dev_err_probe(dev, err: ret, fmt: "Failed to register DMA device\n");
631
632 return 0;
633}
634
635static void d350_remove(struct platform_device *pdev)
636{
637 struct d350 *dmac = platform_get_drvdata(pdev);
638
639 dma_async_device_unregister(device: &dmac->dma);
640}
641
642static const struct of_device_id d350_of_match[] __maybe_unused = {
643 { .compatible = "arm,dma-350" },
644 {}
645};
646MODULE_DEVICE_TABLE(of, d350_of_match);
647
648static struct platform_driver d350_driver = {
649 .driver = {
650 .name = "arm-dma350",
651 .of_match_table = of_match_ptr(d350_of_match),
652 },
653 .probe = d350_probe,
654 .remove = d350_remove,
655};
656module_platform_driver(d350_driver);
657
658MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
659MODULE_DESCRIPTION("Arm DMA-350 driver");
660MODULE_LICENSE("GPL v2");
661

source code of linux/drivers/dma/arm-dma350.c