1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) Ericsson AB 2007-2008
4	* Copyright (C) ST-Ericsson SA 2008-2010
5	* Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
6	* Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
7	*/
8
9	#include <linux/dma-mapping.h>
10	#include <linux/kernel.h>
11	#include <linux/slab.h>
12	#include <linux/export.h>
13	#include <linux/dmaengine.h>
14	#include <linux/platform_device.h>
15	#include <linux/clk.h>
16	#include <linux/delay.h>
17	#include <linux/log2.h>
18	#include <linux/pm.h>
19	#include <linux/pm_runtime.h>
20	#include <linux/err.h>
21	#include <linux/of.h>
22	#include <linux/of_address.h>
23	#include <linux/of_dma.h>
24	#include <linux/amba/bus.h>
25	#include <linux/regulator/consumer.h>
26
27	#include "dmaengine.h"
28	#include "ste_dma40.h"
29	#include "ste_dma40_ll.h"
30
31	/**
32	* struct stedma40_platform_data - Configuration struct for the dma device.
33	*
34	* @disabled_channels: A vector, ending with -1, that marks physical channels
35	* that are for different reasons not available for the driver.
36	* @soft_lli_chans: A vector, that marks physical channels will use LLI by SW
37	* which avoids HW bug that exists in some versions of the controller.
38	* SoftLLI introduces relink overhead that could impact performance for
39	* certain use cases.
40	* @num_of_soft_lli_chans: The number of channels that needs to be configured
41	* to use SoftLLI.
42	* @use_esram_lcla: flag for mapping the lcla into esram region
43	* @num_of_memcpy_chans: The number of channels reserved for memcpy.
44	* @num_of_phy_chans: The number of physical channels implemented in HW.
45	* 0 means reading the number of channels from DMA HW but this is only valid
46	* for 'multiple of 4' channels, like 8.
47	*/
48	struct stedma40_platform_data {
49	int disabled_channels[STEDMA40_MAX_PHYS];
50	int *soft_lli_chans;
51	int num_of_soft_lli_chans;
52	bool use_esram_lcla;
53	int num_of_memcpy_chans;
54	int num_of_phy_chans;
55	};
56
57	#define D40_NAME "dma40"
58
59	#define D40_PHY_CHAN -1
60
61	/* For masking out/in 2 bit channel positions */
62	#define D40_CHAN_POS(chan) (2 * (chan / 2))
63	#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
64
65	/* Maximum iterations taken before giving up suspending a channel */
66	#define D40_SUSPEND_MAX_IT 500
67
68	/* Milliseconds */
69	#define DMA40_AUTOSUSPEND_DELAY 100
70
71	/* Hardware requirement on LCLA alignment */
72	#define LCLA_ALIGNMENT 0x40000
73
74	/* Max number of links per event group */
75	#define D40_LCLA_LINK_PER_EVENT_GRP 128
76	#define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
77
78	/* Max number of logical channels per physical channel */
79	#define D40_MAX_LOG_CHAN_PER_PHY 32
80
81	/* Attempts before giving up to trying to get pages that are aligned */
82	#define MAX_LCLA_ALLOC_ATTEMPTS 256
83
84	/* Bit markings for allocation map */
85	#define D40_ALLOC_FREE BIT(31)
86	#define D40_ALLOC_PHY BIT(30)
87	#define D40_ALLOC_LOG_FREE 0
88
89	#define D40_MEMCPY_MAX_CHANS 8
90
91	/* Reserved event lines for memcpy only. */
92	#define DB8500_DMA_MEMCPY_EV_0 51
93	#define DB8500_DMA_MEMCPY_EV_1 56
94	#define DB8500_DMA_MEMCPY_EV_2 57
95	#define DB8500_DMA_MEMCPY_EV_3 58
96	#define DB8500_DMA_MEMCPY_EV_4 59
97	#define DB8500_DMA_MEMCPY_EV_5 60
98
99	static int dma40_memcpy_channels[] = {
100	DB8500_DMA_MEMCPY_EV_0,
101	DB8500_DMA_MEMCPY_EV_1,
102	DB8500_DMA_MEMCPY_EV_2,
103	DB8500_DMA_MEMCPY_EV_3,
104	DB8500_DMA_MEMCPY_EV_4,
105	DB8500_DMA_MEMCPY_EV_5,
106	};
107
108	/* Default configuration for physical memcpy */
109	static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = {
110	.mode = STEDMA40_MODE_PHYSICAL,
111	.dir = DMA_MEM_TO_MEM,
112
113	.src_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
114	.src_info.psize = STEDMA40_PSIZE_PHY_1,
115	.src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
116
117	.dst_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
118	.dst_info.psize = STEDMA40_PSIZE_PHY_1,
119	.dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
120	};
121
122	/* Default configuration for logical memcpy */
123	static const struct stedma40_chan_cfg dma40_memcpy_conf_log = {
124	.mode = STEDMA40_MODE_LOGICAL,
125	.dir = DMA_MEM_TO_MEM,
126
127	.src_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
128	.src_info.psize = STEDMA40_PSIZE_LOG_1,
129	.src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
130
131	.dst_info.data_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
132	.dst_info.psize = STEDMA40_PSIZE_LOG_1,
133	.dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL,
134	};
135
136	/**
137	* enum d40_command - The different commands and/or statuses.
138	*
139	* @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
140	* @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
141	* @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
142	* @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
143	*/
144	enum d40_command {
145	D40_DMA_STOP = 0,
146	D40_DMA_RUN = 1,
147	D40_DMA_SUSPEND_REQ = 2,
148	D40_DMA_SUSPENDED = 3
149	};
150
151	/*
152	* enum d40_events - The different Event Enables for the event lines.
153	*
154	* @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
155	* @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
156	* @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
157	* @D40_ROUND_EVENTLINE: Status check for event line.
158	*/
159
160	enum d40_events {
161	D40_DEACTIVATE_EVENTLINE = 0,
162	D40_ACTIVATE_EVENTLINE = 1,
163	D40_SUSPEND_REQ_EVENTLINE = 2,
164	D40_ROUND_EVENTLINE = 3
165	};
166
167	/*
168	* These are the registers that has to be saved and later restored
169	* when the DMA hw is powered off.
170	* TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
171	*/
172	static __maybe_unused u32 d40_backup_regs[] = {
173	D40_DREG_LCPA,
174	D40_DREG_LCLA,
175	D40_DREG_PRMSE,
176	D40_DREG_PRMSO,
177	D40_DREG_PRMOE,
178	D40_DREG_PRMOO,
179	};
180
181	#define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
182
183	/*
184	* since 9540 and 8540 has the same HW revision
185	* use v4a for 9540 or earlier
186	* use v4b for 8540 or later
187	* HW revision:
188	* DB8500ed has revision 0
189	* DB8500v1 has revision 2
190	* DB8500v2 has revision 3
191	* AP9540v1 has revision 4
192	* DB8540v1 has revision 4
193	* TODO: Check if all these registers have to be saved/restored on dma40 v4a
194	*/
195	static u32 d40_backup_regs_v4a[] = {
196	D40_DREG_PSEG1,
197	D40_DREG_PSEG2,
198	D40_DREG_PSEG3,
199	D40_DREG_PSEG4,
200	D40_DREG_PCEG1,
201	D40_DREG_PCEG2,
202	D40_DREG_PCEG3,
203	D40_DREG_PCEG4,
204	D40_DREG_RSEG1,
205	D40_DREG_RSEG2,
206	D40_DREG_RSEG3,
207	D40_DREG_RSEG4,
208	D40_DREG_RCEG1,
209	D40_DREG_RCEG2,
210	D40_DREG_RCEG3,
211	D40_DREG_RCEG4,
212	};
213
214	#define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
215
216	static u32 d40_backup_regs_v4b[] = {
217	D40_DREG_CPSEG1,
218	D40_DREG_CPSEG2,
219	D40_DREG_CPSEG3,
220	D40_DREG_CPSEG4,
221	D40_DREG_CPSEG5,
222	D40_DREG_CPCEG1,
223	D40_DREG_CPCEG2,
224	D40_DREG_CPCEG3,
225	D40_DREG_CPCEG4,
226	D40_DREG_CPCEG5,
227	D40_DREG_CRSEG1,
228	D40_DREG_CRSEG2,
229	D40_DREG_CRSEG3,
230	D40_DREG_CRSEG4,
231	D40_DREG_CRSEG5,
232	D40_DREG_CRCEG1,
233	D40_DREG_CRCEG2,
234	D40_DREG_CRCEG3,
235	D40_DREG_CRCEG4,
236	D40_DREG_CRCEG5,
237	};
238
239	#define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
240
241	static __maybe_unused u32 d40_backup_regs_chan[] = {
242	D40_CHAN_REG_SSCFG,
243	D40_CHAN_REG_SSELT,
244	D40_CHAN_REG_SSPTR,
245	D40_CHAN_REG_SSLNK,
246	D40_CHAN_REG_SDCFG,
247	D40_CHAN_REG_SDELT,
248	D40_CHAN_REG_SDPTR,
249	D40_CHAN_REG_SDLNK,
250	};
251
252	#define BACKUP_REGS_SZ_MAX ((BACKUP_REGS_SZ_V4A > BACKUP_REGS_SZ_V4B) ? \
253	BACKUP_REGS_SZ_V4A : BACKUP_REGS_SZ_V4B)
254
255	/**
256	* struct d40_interrupt_lookup - lookup table for interrupt handler
257	*
258	* @src: Interrupt mask register.
259	* @clr: Interrupt clear register.
260	* @is_error: true if this is an error interrupt.
261	* @offset: start delta in the lookup_log_chans in d40_base. If equals to
262	* D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
263	*/
264	struct d40_interrupt_lookup {
265	u32 src;
266	u32 clr;
267	bool is_error;
268	int offset;
269	};
270
271
272	static struct d40_interrupt_lookup il_v4a[] = {
273	{D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
274	{D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
275	{D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
276	{D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
277	{D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
278	{D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
279	{D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
280	{D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
281	{D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
282	{D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
283	};
284
285	static struct d40_interrupt_lookup il_v4b[] = {
286	{D40_DREG_CLCTIS1, D40_DREG_CLCICR1, false, 0},
287	{D40_DREG_CLCTIS2, D40_DREG_CLCICR2, false, 32},
288	{D40_DREG_CLCTIS3, D40_DREG_CLCICR3, false, 64},
289	{D40_DREG_CLCTIS4, D40_DREG_CLCICR4, false, 96},
290	{D40_DREG_CLCTIS5, D40_DREG_CLCICR5, false, 128},
291	{D40_DREG_CLCEIS1, D40_DREG_CLCICR1, true, 0},
292	{D40_DREG_CLCEIS2, D40_DREG_CLCICR2, true, 32},
293	{D40_DREG_CLCEIS3, D40_DREG_CLCICR3, true, 64},
294	{D40_DREG_CLCEIS4, D40_DREG_CLCICR4, true, 96},
295	{D40_DREG_CLCEIS5, D40_DREG_CLCICR5, true, 128},
296	{D40_DREG_CPCTIS, D40_DREG_CPCICR, false, D40_PHY_CHAN},
297	{D40_DREG_CPCEIS, D40_DREG_CPCICR, true, D40_PHY_CHAN},
298	};
299
300	/**
301	* struct d40_reg_val - simple lookup struct
302	*
303	* @reg: The register.
304	* @val: The value that belongs to the register in reg.
305	*/
306	struct d40_reg_val {
307	unsigned int reg;
308	unsigned int val;
309	};
310
311	static __initdata struct d40_reg_val dma_init_reg_v4a[] = {
312	/* Clock every part of the DMA block from start */
313	{ .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
314
315	/* Interrupts on all logical channels */
316	{ .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
317	{ .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
318	{ .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
319	{ .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
320	{ .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
321	{ .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
322	{ .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
323	{ .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
324	{ .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
325	{ .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
326	{ .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
327	{ .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
328	};
329	static __initdata struct d40_reg_val dma_init_reg_v4b[] = {
330	/* Clock every part of the DMA block from start */
331	{ .reg = D40_DREG_GCC, .val = D40_DREG_GCC_ENABLE_ALL},
332
333	/* Interrupts on all logical channels */
334	{ .reg = D40_DREG_CLCMIS1, .val = 0xFFFFFFFF},
335	{ .reg = D40_DREG_CLCMIS2, .val = 0xFFFFFFFF},
336	{ .reg = D40_DREG_CLCMIS3, .val = 0xFFFFFFFF},
337	{ .reg = D40_DREG_CLCMIS4, .val = 0xFFFFFFFF},
338	{ .reg = D40_DREG_CLCMIS5, .val = 0xFFFFFFFF},
339	{ .reg = D40_DREG_CLCICR1, .val = 0xFFFFFFFF},
340	{ .reg = D40_DREG_CLCICR2, .val = 0xFFFFFFFF},
341	{ .reg = D40_DREG_CLCICR3, .val = 0xFFFFFFFF},
342	{ .reg = D40_DREG_CLCICR4, .val = 0xFFFFFFFF},
343	{ .reg = D40_DREG_CLCICR5, .val = 0xFFFFFFFF},
344	{ .reg = D40_DREG_CLCTIS1, .val = 0xFFFFFFFF},
345	{ .reg = D40_DREG_CLCTIS2, .val = 0xFFFFFFFF},
346	{ .reg = D40_DREG_CLCTIS3, .val = 0xFFFFFFFF},
347	{ .reg = D40_DREG_CLCTIS4, .val = 0xFFFFFFFF},
348	{ .reg = D40_DREG_CLCTIS5, .val = 0xFFFFFFFF}
349	};
350
351	/**
352	* struct d40_lli_pool - Structure for keeping LLIs in memory
353	*
354	* @base: Pointer to memory area when the pre_alloc_lli's are not large
355	* enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
356	* pre_alloc_lli is used.
357	* @dma_addr: DMA address, if mapped
358	* @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
359	* @pre_alloc_lli: Pre allocated area for the most common case of transfers,
360	* one buffer to one buffer.
361	*/
362	struct d40_lli_pool {
363	void *base;
364	int size;
365	dma_addr_t dma_addr;
366	/* Space for dst and src, plus an extra for padding */
367	u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
368	};
369
370	/**
371	* struct d40_desc - A descriptor is one DMA job.
372	*
373	* @lli_phy: LLI settings for physical channel. Both src and dst=
374	* points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
375	* lli_len equals one.
376	* @lli_log: Same as above but for logical channels.
377	* @lli_pool: The pool with two entries pre-allocated.
378	* @lli_len: Number of llis of current descriptor.
379	* @lli_current: Number of transferred llis.
380	* @lcla_alloc: Number of LCLA entries allocated.
381	* @txd: DMA engine struct. Used for among other things for communication
382	* during a transfer.
383	* @node: List entry.
384	* @is_in_client_list: true if the client owns this descriptor.
385	* @cyclic: true if this is a cyclic job
386	*
387	* This descriptor is used for both logical and physical transfers.
388	*/
389	struct d40_desc {
390	/* LLI physical */
391	struct d40_phy_lli_bidir lli_phy;
392	/* LLI logical */
393	struct d40_log_lli_bidir lli_log;
394
395	struct d40_lli_pool lli_pool;
396	int lli_len;
397	int lli_current;
398	int lcla_alloc;
399
400	struct dma_async_tx_descriptor txd;
401	struct list_head node;
402
403	bool is_in_client_list;
404	bool cyclic;
405	};
406
407	/**
408	* struct d40_lcla_pool - LCLA pool settings and data.
409	*
410	* @base: The virtual address of LCLA. 18 bit aligned.
411	* @dma_addr: DMA address, if mapped
412	* @base_unaligned: The original kmalloc pointer, if kmalloc is used.
413	* This pointer is only there for clean-up on error.
414	* @pages: The number of pages needed for all physical channels.
415	* Only used later for clean-up on error
416	* @lock: Lock to protect the content in this struct.
417	* @alloc_map: big map over which LCLA entry is own by which job.
418	*/
419	struct d40_lcla_pool {
420	void *base;
421	dma_addr_t dma_addr;
422	void *base_unaligned;
423	int pages;
424	spinlock_t lock;
425	struct d40_desc **alloc_map;
426	};
427
428	/**
429	* struct d40_phy_res - struct for handling eventlines mapped to physical
430	* channels.
431	*
432	* @lock: A lock protection this entity.
433	* @reserved: True if used by secure world or otherwise.
434	* @num: The physical channel number of this entity.
435	* @allocated_src: Bit mapped to show which src event line's are mapped to
436	* this physical channel. Can also be free or physically allocated.
437	* @allocated_dst: Same as for src but is dst.
438	* allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
439	* event line number.
440	* @use_soft_lli: To mark if the linked lists of channel are managed by SW.
441	*/
442	struct d40_phy_res {
443	spinlock_t lock;
444	bool reserved;
445	int num;
446	u32 allocated_src;
447	u32 allocated_dst;
448	bool use_soft_lli;
449	};
450
451	struct d40_base;
452
453	/**
454	* struct d40_chan - Struct that describes a channel.
455	*
456	* @lock: A spinlock to protect this struct.
457	* @log_num: The logical number, if any of this channel.
458	* @pending_tx: The number of pending transfers. Used between interrupt handler
459	* and tasklet.
460	* @busy: Set to true when transfer is ongoing on this channel.
461	* @phy_chan: Pointer to physical channel which this instance runs on. If this
462	* point is NULL, then the channel is not allocated.
463	* @chan: DMA engine handle.
464	* @tasklet: Tasklet that gets scheduled from interrupt context to complete a
465	* transfer and call client callback.
466	* @client: Cliented owned descriptor list.
467	* @pending_queue: Submitted jobs, to be issued by issue_pending()
468	* @active: Active descriptor.
469	* @done: Completed jobs
470	* @queue: Queued jobs.
471	* @prepare_queue: Prepared jobs.
472	* @dma_cfg: The client configuration of this dma channel.
473	* @slave_config: DMA slave configuration.
474	* @configured: whether the dma_cfg configuration is valid
475	* @base: Pointer to the device instance struct.
476	* @src_def_cfg: Default cfg register setting for src.
477	* @dst_def_cfg: Default cfg register setting for dst.
478	* @log_def: Default logical channel settings.
479	* @lcpa: Pointer to dst and src lcpa settings.
480	* @runtime_addr: runtime configured address.
481	* @runtime_direction: runtime configured direction.
482	*
483	* This struct can either "be" a logical or a physical channel.
484	*/
485	struct d40_chan {
486	spinlock_t lock;
487	int log_num;
488	int pending_tx;
489	bool busy;
490	struct d40_phy_res *phy_chan;
491	struct dma_chan chan;
492	struct tasklet_struct tasklet;
493	struct list_head client;
494	struct list_head pending_queue;
495	struct list_head active;
496	struct list_head done;
497	struct list_head queue;
498	struct list_head prepare_queue;
499	struct stedma40_chan_cfg dma_cfg;
500	struct dma_slave_config slave_config;
501	bool configured;
502	struct d40_base *base;
503	/* Default register configurations */
504	u32 src_def_cfg;
505	u32 dst_def_cfg;
506	struct d40_def_lcsp log_def;
507	struct d40_log_lli_full *lcpa;
508	/* Runtime reconfiguration */
509	dma_addr_t runtime_addr;
510	enum dma_transfer_direction runtime_direction;
511	};
512
513	/**
514	* struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
515	* controller
516	*
517	* @backup: the pointer to the registers address array for backup
518	* @backup_size: the size of the registers address array for backup
519	* @realtime_en: the realtime enable register
520	* @realtime_clear: the realtime clear register
521	* @high_prio_en: the high priority enable register
522	* @high_prio_clear: the high priority clear register
523	* @interrupt_en: the interrupt enable register
524	* @interrupt_clear: the interrupt clear register
525	* @il: the pointer to struct d40_interrupt_lookup
526	* @il_size: the size of d40_interrupt_lookup array
527	* @init_reg: the pointer to the struct d40_reg_val
528	* @init_reg_size: the size of d40_reg_val array
529	*/
530	struct d40_gen_dmac {
531	u32 *backup;
532	u32 backup_size;
533	u32 realtime_en;
534	u32 realtime_clear;
535	u32 high_prio_en;
536	u32 high_prio_clear;
537	u32 interrupt_en;
538	u32 interrupt_clear;
539	struct d40_interrupt_lookup *il;
540	u32 il_size;
541	struct d40_reg_val *init_reg;
542	u32 init_reg_size;
543	};
544
545	/**
546	* struct d40_base - The big global struct, one for each probe'd instance.
547	*
548	* @interrupt_lock: Lock used to make sure one interrupt is handle a time.
549	* @execmd_lock: Lock for execute command usage since several channels share
550	* the same physical register.
551	* @dev: The device structure.
552	* @virtbase: The virtual base address of the DMA's register.
553	* @rev: silicon revision detected.
554	* @clk: Pointer to the DMA clock structure.
555	* @irq: The IRQ number.
556	* @num_memcpy_chans: The number of channels used for memcpy (mem-to-mem
557	* transfers).
558	* @num_phy_chans: The number of physical channels. Read from HW. This
559	* is the number of available channels for this driver, not counting "Secure
560	* mode" allocated physical channels.
561	* @num_log_chans: The number of logical channels. Calculated from
562	* num_phy_chans.
563	* @dma_both: dma_device channels that can do both memcpy and slave transfers.
564	* @dma_slave: dma_device channels that can do only do slave transfers.
565	* @dma_memcpy: dma_device channels that can do only do memcpy transfers.
566	* @phy_chans: Room for all possible physical channels in system.
567	* @log_chans: Room for all possible logical channels in system.
568	* @lookup_log_chans: Used to map interrupt number to logical channel. Points
569	* to log_chans entries.
570	* @lookup_phy_chans: Used to map interrupt number to physical channel. Points
571	* to phy_chans entries.
572	* @plat_data: Pointer to provided platform_data which is the driver
573	* configuration.
574	* @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
575	* @phy_res: Vector containing all physical channels.
576	* @lcla_pool: lcla pool settings and data.
577	* @lcpa_base: The virtual mapped address of LCPA.
578	* @phy_lcpa: The physical address of the LCPA.
579	* @lcpa_size: The size of the LCPA area.
580	* @desc_slab: cache for descriptors.
581	* @reg_val_backup: Here the values of some hardware registers are stored
582	* before the DMA is powered off. They are restored when the power is back on.
583	* @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
584	* later
585	* @reg_val_backup_chan: Backup data for standard channel parameter registers.
586	* @regs_interrupt: Scratch space for registers during interrupt.
587	* @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
588	* @gen_dmac: the struct for generic registers values to represent u8500/8540
589	* DMA controller
590	*/
591	struct d40_base {
592	spinlock_t interrupt_lock;
593	spinlock_t execmd_lock;
594	struct device *dev;
595	void __iomem *virtbase;
596	u8 rev:4;
597	struct clk *clk;
598	int irq;
599	int num_memcpy_chans;
600	int num_phy_chans;
601	int num_log_chans;
602	struct dma_device dma_both;
603	struct dma_device dma_slave;
604	struct dma_device dma_memcpy;
605	struct d40_chan *phy_chans;
606	struct d40_chan *log_chans;
607	struct d40_chan **lookup_log_chans;
608	struct d40_chan **lookup_phy_chans;
609	struct stedma40_platform_data *plat_data;
610	struct regulator *lcpa_regulator;
611	/* Physical half channels */
612	struct d40_phy_res *phy_res;
613	struct d40_lcla_pool lcla_pool;
614	void *lcpa_base;
615	dma_addr_t phy_lcpa;
616	resource_size_t lcpa_size;
617	struct kmem_cache *desc_slab;
618	u32 reg_val_backup[BACKUP_REGS_SZ];
619	u32 reg_val_backup_v4[BACKUP_REGS_SZ_MAX];
620	u32 *reg_val_backup_chan;
621	u32 *regs_interrupt;
622	u16 gcc_pwr_off_mask;
623	struct d40_gen_dmac gen_dmac;
624	};
625
626	static struct device *chan2dev(struct d40_chan *d40c)
627	{
628	return &d40c->chan.dev->device;
629	}
630
631	static bool chan_is_physical(struct d40_chan *chan)
632	{
633	return chan->log_num == D40_PHY_CHAN;
634	}
635
636	static bool chan_is_logical(struct d40_chan *chan)
637	{
638	return !chan_is_physical(chan);
639	}
640
641	static void __iomem *chan_base(struct d40_chan *chan)
642	{
643	return chan->base->virtbase + D40_DREG_PCBASE +
644	chan->phy_chan->num * D40_DREG_PCDELTA;
645	}
646
647	#define d40_err(dev, format, arg...) \
648	dev_err(dev, "[%s] " format, __func__, ## arg)
649
650	#define chan_err(d40c, format, arg...) \
651	d40_err(chan2dev(d40c), format, ## arg)
652
653	static int d40_set_runtime_config_write(struct dma_chan *chan,
654	struct dma_slave_config *config,
655	enum dma_transfer_direction direction);
656
657	static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
658	int lli_len)
659	{
660	bool is_log = chan_is_logical(chan: d40c);
661	u32 align;
662	void *base;
663
664	if (is_log)
665	align = sizeof(struct d40_log_lli);
666	else
667	align = sizeof(struct d40_phy_lli);
668
669	if (lli_len == 1) {
670	base = d40d->lli_pool.pre_alloc_lli;
671	d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
672	d40d->lli_pool.base = NULL;
673	} else {
674	d40d->lli_pool.size = lli_len * 2 * align;
675
676	base = kmalloc(size: d40d->lli_pool.size + align, GFP_NOWAIT);
677	d40d->lli_pool.base = base;
678
679	if (d40d->lli_pool.base == NULL)
680	return -ENOMEM;
681	}
682
683	if (is_log) {
684	d40d->lli_log.src = PTR_ALIGN(base, align);
685	d40d->lli_log.dst = d40d->lli_log.src + lli_len;
686
687	d40d->lli_pool.dma_addr = 0;
688	} else {
689	d40d->lli_phy.src = PTR_ALIGN(base, align);
690	d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
691
692	d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
693	d40d->lli_phy.src,
694	d40d->lli_pool.size,
695	DMA_TO_DEVICE);
696
697	if (dma_mapping_error(dev: d40c->base->dev,
698	dma_addr: d40d->lli_pool.dma_addr)) {
699	kfree(objp: d40d->lli_pool.base);
700	d40d->lli_pool.base = NULL;
701	d40d->lli_pool.dma_addr = 0;
702	return -ENOMEM;
703	}
704	}
705
706	return 0;
707	}
708
709	static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
710	{
711	if (d40d->lli_pool.dma_addr)
712	dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
713	d40d->lli_pool.size, DMA_TO_DEVICE);
714
715	kfree(objp: d40d->lli_pool.base);
716	d40d->lli_pool.base = NULL;
717	d40d->lli_pool.size = 0;
718	d40d->lli_log.src = NULL;
719	d40d->lli_log.dst = NULL;
720	d40d->lli_phy.src = NULL;
721	d40d->lli_phy.dst = NULL;
722	}
723
724	static int d40_lcla_alloc_one(struct d40_chan *d40c,
725	struct d40_desc *d40d)
726	{
727	unsigned long flags;
728	int i;
729	int ret = -EINVAL;
730
731	spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
732
733	/*
734	* Allocate both src and dst at the same time, therefore the half
735	* start on 1 since 0 can't be used since zero is used as end marker.
736	*/
737	for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
738	int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
739
740	if (!d40c->base->lcla_pool.alloc_map[idx]) {
741	d40c->base->lcla_pool.alloc_map[idx] = d40d;
742	d40d->lcla_alloc++;
743	ret = i;
744	break;
745	}
746	}
747
748	spin_unlock_irqrestore(lock: &d40c->base->lcla_pool.lock, flags);
749
750	return ret;
751	}
752
753	static int d40_lcla_free_all(struct d40_chan *d40c,
754	struct d40_desc *d40d)
755	{
756	unsigned long flags;
757	int i;
758	int ret = -EINVAL;
759
760	if (chan_is_physical(chan: d40c))
761	return 0;
762
763	spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
764
765	for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
766	int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i;
767
768	if (d40c->base->lcla_pool.alloc_map[idx] == d40d) {
769	d40c->base->lcla_pool.alloc_map[idx] = NULL;
770	d40d->lcla_alloc--;
771	if (d40d->lcla_alloc == 0) {
772	ret = 0;
773	break;
774	}
775	}
776	}
777
778	spin_unlock_irqrestore(lock: &d40c->base->lcla_pool.lock, flags);
779
780	return ret;
781
782	}
783
784	static void d40_desc_remove(struct d40_desc *d40d)
785	{
786	list_del(entry: &d40d->node);
787	}
788
789	static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
790	{
791	struct d40_desc *desc = NULL;
792
793	if (!list_empty(head: &d40c->client)) {
794	struct d40_desc *d;
795	struct d40_desc *_d;
796
797	list_for_each_entry_safe(d, _d, &d40c->client, node) {
798	if (async_tx_test_ack(tx: &d->txd)) {
799	d40_desc_remove(d40d: d);
800	desc = d;
801	memset(desc, 0, sizeof(*desc));
802	break;
803	}
804	}
805	}
806
807	if (!desc)
808	desc = kmem_cache_zalloc(k: d40c->base->desc_slab, GFP_NOWAIT);
809
810	if (desc)
811	INIT_LIST_HEAD(list: &desc->node);
812
813	return desc;
814	}
815
816	static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
817	{
818
819	d40_pool_lli_free(d40c, d40d);
820	d40_lcla_free_all(d40c, d40d);
821	kmem_cache_free(s: d40c->base->desc_slab, objp: d40d);
822	}
823
824	static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
825	{
826	list_add_tail(new: &desc->node, head: &d40c->active);
827	}
828
829	static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
830	{
831	struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
832	struct d40_phy_lli *lli_src = desc->lli_phy.src;
833	void __iomem *base = chan_base(chan);
834
835	writel(val: lli_src->reg_cfg, addr: base + D40_CHAN_REG_SSCFG);
836	writel(val: lli_src->reg_elt, addr: base + D40_CHAN_REG_SSELT);
837	writel(val: lli_src->reg_ptr, addr: base + D40_CHAN_REG_SSPTR);
838	writel(val: lli_src->reg_lnk, addr: base + D40_CHAN_REG_SSLNK);
839
840	writel(val: lli_dst->reg_cfg, addr: base + D40_CHAN_REG_SDCFG);
841	writel(val: lli_dst->reg_elt, addr: base + D40_CHAN_REG_SDELT);
842	writel(val: lli_dst->reg_ptr, addr: base + D40_CHAN_REG_SDPTR);
843	writel(val: lli_dst->reg_lnk, addr: base + D40_CHAN_REG_SDLNK);
844	}
845
846	static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc)
847	{
848	list_add_tail(new: &desc->node, head: &d40c->done);
849	}
850
851	static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
852	{
853	struct d40_lcla_pool *pool = &chan->base->lcla_pool;
854	struct d40_log_lli_bidir *lli = &desc->lli_log;
855	int lli_current = desc->lli_current;
856	int lli_len = desc->lli_len;
857	bool cyclic = desc->cyclic;
858	int curr_lcla = -EINVAL;
859	int first_lcla = 0;
860	bool use_esram_lcla = chan->base->plat_data->use_esram_lcla;
861	bool linkback;
862
863	/*
864	* We may have partially running cyclic transfers, in case we did't get
865	* enough LCLA entries.
866	*/
867	linkback = cyclic && lli_current == 0;
868
869	/*
870	* For linkback, we need one LCLA even with only one link, because we
871	* can't link back to the one in LCPA space
872	*/
873	if (linkback || (lli_len - lli_current > 1)) {
874	/*
875	* If the channel is expected to use only soft_lli don't
876	* allocate a lcla. This is to avoid a HW issue that exists
877	* in some controller during a peripheral to memory transfer
878	* that uses linked lists.
879	*/
880	if (!(chan->phy_chan->use_soft_lli &&
881	chan->dma_cfg.dir == DMA_DEV_TO_MEM))
882	curr_lcla = d40_lcla_alloc_one(d40c: chan, d40d: desc);
883
884	first_lcla = curr_lcla;
885	}
886
887	/*
888	* For linkback, we normally load the LCPA in the loop since we need to
889	* link it to the second LCLA and not the first. However, if we
890	* couldn't even get a first LCLA, then we have to run in LCPA and
891	* reload manually.
892	*/
893	if (!linkback || curr_lcla == -EINVAL) {
894	unsigned int flags = 0;
895
896	if (curr_lcla == -EINVAL)
897	flags |= LLI_TERM_INT;
898
899	d40_log_lli_lcpa_write(lcpa: chan->lcpa,
900	lli_dst: &lli->dst[lli_current],
901	lli_src: &lli->src[lli_current],
902	next: curr_lcla,
903	flags);
904	lli_current++;
905	}
906
907	if (curr_lcla < 0)
908	goto set_current;
909
910	for (; lli_current < lli_len; lli_current++) {
911	unsigned int lcla_offset = chan->phy_chan->num * 1024 +
912	8 * curr_lcla * 2;
913	struct d40_log_lli *lcla = pool->base + lcla_offset;
914	unsigned int flags = 0;
915	int next_lcla;
916
917	if (lli_current + 1 < lli_len)
918	next_lcla = d40_lcla_alloc_one(d40c: chan, d40d: desc);
919	else
920	next_lcla = linkback ? first_lcla : -EINVAL;
921
922	if (cyclic || next_lcla == -EINVAL)
923	flags |= LLI_TERM_INT;
924
925	if (linkback && curr_lcla == first_lcla) {
926	/* First link goes in both LCPA and LCLA */
927	d40_log_lli_lcpa_write(lcpa: chan->lcpa,
928	lli_dst: &lli->dst[lli_current],
929	lli_src: &lli->src[lli_current],
930	next: next_lcla, flags);
931	}
932
933	/*
934	* One unused LCLA in the cyclic case if the very first
935	* next_lcla fails...
936	*/
937	d40_log_lli_lcla_write(lcla,
938	lli_dst: &lli->dst[lli_current],
939	lli_src: &lli->src[lli_current],
940	next: next_lcla, flags);
941
942	/*
943	* Cache maintenance is not needed if lcla is
944	* mapped in esram
945	*/
946	if (!use_esram_lcla) {
947	dma_sync_single_range_for_device(dev: chan->base->dev,
948	addr: pool->dma_addr, offset: lcla_offset,
949	size: 2 * sizeof(struct d40_log_lli),
950	dir: DMA_TO_DEVICE);
951	}
952	curr_lcla = next_lcla;
953
954	if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
955	lli_current++;
956	break;
957	}
958	}
959	set_current:
960	desc->lli_current = lli_current;
961	}
962
963	static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
964	{
965	if (chan_is_physical(chan: d40c)) {
966	d40_phy_lli_load(chan: d40c, desc: d40d);
967	d40d->lli_current = d40d->lli_len;
968	} else
969	d40_log_lli_to_lcxa(chan: d40c, desc: d40d);
970	}
971
972	static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
973	{
974	return list_first_entry_or_null(&d40c->active, struct d40_desc, node);
975	}
976
977	/* remove desc from current queue and add it to the pending_queue */
978	static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
979	{
980	d40_desc_remove(d40d: desc);
981	desc->is_in_client_list = false;
982	list_add_tail(new: &desc->node, head: &d40c->pending_queue);
983	}
984
985	static struct d40_desc *d40_first_pending(struct d40_chan *d40c)
986	{
987	return list_first_entry_or_null(&d40c->pending_queue, struct d40_desc,
988	node);
989	}
990
991	static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
992	{
993	return list_first_entry_or_null(&d40c->queue, struct d40_desc, node);
994	}
995
996	static struct d40_desc *d40_first_done(struct d40_chan *d40c)
997	{
998	return list_first_entry_or_null(&d40c->done, struct d40_desc, node);
999	}
1000
1001	static int d40_psize_2_burst_size(bool is_log, int psize)
1002	{
1003	if (is_log) {
1004	if (psize == STEDMA40_PSIZE_LOG_1)
1005	return 1;
1006	} else {
1007	if (psize == STEDMA40_PSIZE_PHY_1)
1008	return 1;
1009	}
1010
1011	return 2 << psize;
1012	}
1013
1014	/*
1015	* The dma only supports transmitting packages up to
1016	* STEDMA40_MAX_SEG_SIZE * data_width, where data_width is stored in Bytes.
1017	*
1018	* Calculate the total number of dma elements required to send the entire sg list.
1019	*/
1020	static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
1021	{
1022	int dmalen;
1023	u32 max_w = max(data_width1, data_width2);
1024	u32 min_w = min(data_width1, data_width2);
1025	u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE * min_w, max_w);
1026
1027	if (seg_max > STEDMA40_MAX_SEG_SIZE)
1028	seg_max -= max_w;
1029
1030	if (!IS_ALIGNED(size, max_w))
1031	return -EINVAL;
1032
1033	if (size <= seg_max)
1034	dmalen = 1;
1035	else {
1036	dmalen = size / seg_max;
1037	if (dmalen * seg_max < size)
1038	dmalen++;
1039	}
1040	return dmalen;
1041	}
1042
1043	static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
1044	u32 data_width1, u32 data_width2)
1045	{
1046	struct scatterlist *sg;
1047	int i;
1048	int len = 0;
1049	int ret;
1050
1051	for_each_sg(sgl, sg, sg_len, i) {
1052	ret = d40_size_2_dmalen(sg_dma_len(sg),
1053	data_width1, data_width2);
1054	if (ret < 0)
1055	return ret;
1056	len += ret;
1057	}
1058	return len;
1059	}
1060
1061	static int __d40_execute_command_phy(struct d40_chan *d40c,
1062	enum d40_command command)
1063	{
1064	u32 status;
1065	int i;
1066	void __iomem *active_reg;
1067	int ret = 0;
1068	unsigned long flags;
1069	u32 wmask;
1070
1071	if (command == D40_DMA_STOP) {
1072	ret = __d40_execute_command_phy(d40c, command: D40_DMA_SUSPEND_REQ);
1073	if (ret)
1074	return ret;
1075	}
1076
1077	spin_lock_irqsave(&d40c->base->execmd_lock, flags);
1078
1079	if (d40c->phy_chan->num % 2 == 0)
1080	active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1081	else
1082	active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1083
1084	if (command == D40_DMA_SUSPEND_REQ) {
1085	status = (readl(addr: active_reg) &
1086	D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1087	D40_CHAN_POS(d40c->phy_chan->num);
1088
1089	if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1090	goto unlock;
1091	}
1092
1093	wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
1094	writel(val: wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
1095	addr: active_reg);
1096
1097	if (command == D40_DMA_SUSPEND_REQ) {
1098
1099	for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
1100	status = (readl(addr: active_reg) &
1101	D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1102	D40_CHAN_POS(d40c->phy_chan->num);
1103
1104	cpu_relax();
1105	/*
1106	* Reduce the number of bus accesses while
1107	* waiting for the DMA to suspend.
1108	*/
1109	udelay(3);
1110
1111	if (status == D40_DMA_STOP ||
1112	status == D40_DMA_SUSPENDED)
1113	break;
1114	}
1115
1116	if (i == D40_SUSPEND_MAX_IT) {
1117	chan_err(d40c,
1118	"unable to suspend the chl %d (log: %d) status %x\n",
1119	d40c->phy_chan->num, d40c->log_num,
1120	status);
1121	dump_stack();
1122	ret = -EBUSY;
1123	}
1124
1125	}
1126	unlock:
1127	spin_unlock_irqrestore(lock: &d40c->base->execmd_lock, flags);
1128	return ret;
1129	}
1130
1131	static void d40_term_all(struct d40_chan *d40c)
1132	{
1133	struct d40_desc *d40d;
1134	struct d40_desc *_d;
1135
1136	/* Release completed descriptors */
1137	while ((d40d = d40_first_done(d40c))) {
1138	d40_desc_remove(d40d);
1139	d40_desc_free(d40c, d40d);
1140	}
1141
1142	/* Release active descriptors */
1143	while ((d40d = d40_first_active_get(d40c))) {
1144	d40_desc_remove(d40d);
1145	d40_desc_free(d40c, d40d);
1146	}
1147
1148	/* Release queued descriptors waiting for transfer */
1149	while ((d40d = d40_first_queued(d40c))) {
1150	d40_desc_remove(d40d);
1151	d40_desc_free(d40c, d40d);
1152	}
1153
1154	/* Release pending descriptors */
1155	while ((d40d = d40_first_pending(d40c))) {
1156	d40_desc_remove(d40d);
1157	d40_desc_free(d40c, d40d);
1158	}
1159
1160	/* Release client owned descriptors */
1161	if (!list_empty(head: &d40c->client))
1162	list_for_each_entry_safe(d40d, _d, &d40c->client, node) {
1163	d40_desc_remove(d40d);
1164	d40_desc_free(d40c, d40d);
1165	}
1166
1167	/* Release descriptors in prepare queue */
1168	if (!list_empty(head: &d40c->prepare_queue))
1169	list_for_each_entry_safe(d40d, _d,
1170	&d40c->prepare_queue, node) {
1171	d40_desc_remove(d40d);
1172	d40_desc_free(d40c, d40d);
1173	}
1174
1175	d40c->pending_tx = 0;
1176	}
1177
1178	static void __d40_config_set_event(struct d40_chan *d40c,
1179	enum d40_events event_type, u32 event,
1180	int reg)
1181	{
1182	void __iomem *addr = chan_base(chan: d40c) + reg;
1183	int tries;
1184	u32 status;
1185
1186	switch (event_type) {
1187
1188	case D40_DEACTIVATE_EVENTLINE:
1189
1190	writel(val: (D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
1191	| ~D40_EVENTLINE_MASK(event), addr);
1192	break;
1193
1194	case D40_SUSPEND_REQ_EVENTLINE:
1195	status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1196	D40_EVENTLINE_POS(event);
1197
1198	if (status == D40_DEACTIVATE_EVENTLINE ||
1199	status == D40_SUSPEND_REQ_EVENTLINE)
1200	break;
1201
1202	writel(val: (D40_SUSPEND_REQ_EVENTLINE << D40_EVENTLINE_POS(event))
1203	| ~D40_EVENTLINE_MASK(event), addr);
1204
1205	for (tries = 0 ; tries < D40_SUSPEND_MAX_IT; tries++) {
1206
1207	status = (readl(addr) & D40_EVENTLINE_MASK(event)) >>
1208	D40_EVENTLINE_POS(event);
1209
1210	cpu_relax();
1211	/*
1212	* Reduce the number of bus accesses while
1213	* waiting for the DMA to suspend.
1214	*/
1215	udelay(3);
1216
1217	if (status == D40_DEACTIVATE_EVENTLINE)
1218	break;
1219	}
1220
1221	if (tries == D40_SUSPEND_MAX_IT) {
1222	chan_err(d40c,
1223	"unable to stop the event_line chl %d (log: %d)"
1224	"status %x\n", d40c->phy_chan->num,
1225	d40c->log_num, status);
1226	}
1227	break;
1228
1229	case D40_ACTIVATE_EVENTLINE:
1230	/*
1231	* The hardware sometimes doesn't register the enable when src and dst
1232	* event lines are active on the same logical channel. Retry to ensure
1233	* it does. Usually only one retry is sufficient.
1234	*/
1235	tries = 100;
1236	while (--tries) {
1237	writel(val: (D40_ACTIVATE_EVENTLINE <<
1238	D40_EVENTLINE_POS(event)) |
1239	~D40_EVENTLINE_MASK(event), addr);
1240
1241	if (readl(addr) & D40_EVENTLINE_MASK(event))
1242	break;
1243	}
1244
1245	if (tries != 99)
1246	dev_dbg(chan2dev(d40c),
1247	"[%s] workaround enable S%cLNK (%d tries)\n",
1248	__func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
1249	100 - tries);
1250
1251	WARN_ON(!tries);
1252	break;
1253
1254	case D40_ROUND_EVENTLINE:
1255	BUG();
1256	break;
1257
1258	}
1259	}
1260
1261	static void d40_config_set_event(struct d40_chan *d40c,
1262	enum d40_events event_type)
1263	{
1264	u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
1265
1266	/* Enable event line connected to device (or memcpy) */
1267	if ((d40c->dma_cfg.dir == DMA_DEV_TO_MEM) ||
1268	(d40c->dma_cfg.dir == DMA_DEV_TO_DEV))
1269	__d40_config_set_event(d40c, event_type, event,
1270	D40_CHAN_REG_SSLNK);
1271
1272	if (d40c->dma_cfg.dir != DMA_DEV_TO_MEM)
1273	__d40_config_set_event(d40c, event_type, event,
1274	D40_CHAN_REG_SDLNK);
1275	}
1276
1277	static u32 d40_chan_has_events(struct d40_chan *d40c)
1278	{
1279	void __iomem *chanbase = chan_base(chan: d40c);
1280	u32 val;
1281
1282	val = readl(addr: chanbase + D40_CHAN_REG_SSLNK);
1283	val |= readl(addr: chanbase + D40_CHAN_REG_SDLNK);
1284
1285	return val;
1286	}
1287
1288	static int
1289	__d40_execute_command_log(struct d40_chan *d40c, enum d40_command command)
1290	{
1291	unsigned long flags;
1292	int ret = 0;
1293	u32 active_status;
1294	void __iomem *active_reg;
1295
1296	if (d40c->phy_chan->num % 2 == 0)
1297	active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1298	else
1299	active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1300
1301
1302	spin_lock_irqsave(&d40c->phy_chan->lock, flags);
1303
1304	switch (command) {
1305	case D40_DMA_STOP:
1306	case D40_DMA_SUSPEND_REQ:
1307
1308	active_status = (readl(addr: active_reg) &
1309	D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1310	D40_CHAN_POS(d40c->phy_chan->num);
1311
1312	if (active_status == D40_DMA_RUN)
1313	d40_config_set_event(d40c, event_type: D40_SUSPEND_REQ_EVENTLINE);
1314	else
1315	d40_config_set_event(d40c, event_type: D40_DEACTIVATE_EVENTLINE);
1316
1317	if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP))
1318	ret = __d40_execute_command_phy(d40c, command);
1319
1320	break;
1321
1322	case D40_DMA_RUN:
1323
1324	d40_config_set_event(d40c, event_type: D40_ACTIVATE_EVENTLINE);
1325	ret = __d40_execute_command_phy(d40c, command);
1326	break;
1327
1328	case D40_DMA_SUSPENDED:
1329	BUG();
1330	break;
1331	}
1332
1333	spin_unlock_irqrestore(lock: &d40c->phy_chan->lock, flags);
1334	return ret;
1335	}
1336
1337	static int d40_channel_execute_command(struct d40_chan *d40c,
1338	enum d40_command command)
1339	{
1340	if (chan_is_logical(chan: d40c))
1341	return __d40_execute_command_log(d40c, command);
1342	else
1343	return __d40_execute_command_phy(d40c, command);
1344	}
1345
1346	static u32 d40_get_prmo(struct d40_chan *d40c)
1347	{
1348	static const unsigned int phy_map[] = {
1349	[STEDMA40_PCHAN_BASIC_MODE]
1350	= D40_DREG_PRMO_PCHAN_BASIC,
1351	[STEDMA40_PCHAN_MODULO_MODE]
1352	= D40_DREG_PRMO_PCHAN_MODULO,
1353	[STEDMA40_PCHAN_DOUBLE_DST_MODE]
1354	= D40_DREG_PRMO_PCHAN_DOUBLE_DST,
1355	};
1356	static const unsigned int log_map[] = {
1357	[STEDMA40_LCHAN_SRC_PHY_DST_LOG]
1358	= D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
1359	[STEDMA40_LCHAN_SRC_LOG_DST_PHY]
1360	= D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
1361	[STEDMA40_LCHAN_SRC_LOG_DST_LOG]
1362	= D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
1363	};
1364
1365	if (chan_is_physical(chan: d40c))
1366	return phy_map[d40c->dma_cfg.mode_opt];
1367	else
1368	return log_map[d40c->dma_cfg.mode_opt];
1369	}
1370
1371	static void d40_config_write(struct d40_chan *d40c)
1372	{
1373	u32 addr_base;
1374	u32 var;
1375
1376	/* Odd addresses are even addresses + 4 */
1377	addr_base = (d40c->phy_chan->num % 2) * 4;
1378	/* Setup channel mode to logical or physical */
1379	var = ((u32)(chan_is_logical(chan: d40c)) + 1) <<
1380	D40_CHAN_POS(d40c->phy_chan->num);
1381	writel(val: var, addr: d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
1382
1383	/* Setup operational mode option register */
1384	var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
1385
1386	writel(val: var, addr: d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
1387
1388	if (chan_is_logical(chan: d40c)) {
1389	int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
1390	& D40_SREG_ELEM_LOG_LIDX_MASK;
1391	void __iomem *chanbase = chan_base(chan: d40c);
1392
1393	/* Set default config for CFG reg */
1394	writel(val: d40c->src_def_cfg, addr: chanbase + D40_CHAN_REG_SSCFG);
1395	writel(val: d40c->dst_def_cfg, addr: chanbase + D40_CHAN_REG_SDCFG);
1396
1397	/* Set LIDX for lcla */
1398	writel(val: lidx, addr: chanbase + D40_CHAN_REG_SSELT);
1399	writel(val: lidx, addr: chanbase + D40_CHAN_REG_SDELT);
1400
1401	/* Clear LNK which will be used by d40_chan_has_events() */
1402	writel(val: 0, addr: chanbase + D40_CHAN_REG_SSLNK);
1403	writel(val: 0, addr: chanbase + D40_CHAN_REG_SDLNK);
1404	}
1405	}
1406
1407	static u32 d40_residue(struct d40_chan *d40c)
1408	{
1409	u32 num_elt;
1410
1411	if (chan_is_logical(chan: d40c))
1412	num_elt = (readl(addr: &d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
1413	>> D40_MEM_LCSP2_ECNT_POS;
1414	else {
1415	u32 val = readl(addr: chan_base(chan: d40c) + D40_CHAN_REG_SDELT);
1416	num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
1417	>> D40_SREG_ELEM_PHY_ECNT_POS;
1418	}
1419
1420	return num_elt * d40c->dma_cfg.dst_info.data_width;
1421	}
1422
1423	static bool d40_tx_is_linked(struct d40_chan *d40c)
1424	{
1425	bool is_link;
1426
1427	if (chan_is_logical(chan: d40c))
1428	is_link = readl(addr: &d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
1429	else
1430	is_link = readl(addr: chan_base(chan: d40c) + D40_CHAN_REG_SDLNK)
1431	& D40_SREG_LNK_PHYS_LNK_MASK;
1432
1433	return is_link;
1434	}
1435
1436	static int d40_pause(struct dma_chan *chan)
1437	{
1438	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
1439	int res = 0;
1440	unsigned long flags;
1441
1442	if (d40c->phy_chan == NULL) {
1443	chan_err(d40c, "Channel is not allocated!\n");
1444	return -EINVAL;
1445	}
1446
1447	if (!d40c->busy)
1448	return 0;
1449
1450	spin_lock_irqsave(&d40c->lock, flags);
1451	pm_runtime_get_sync(dev: d40c->base->dev);
1452
1453	res = d40_channel_execute_command(d40c, command: D40_DMA_SUSPEND_REQ);
1454
1455	pm_runtime_mark_last_busy(dev: d40c->base->dev);
1456	pm_runtime_put_autosuspend(dev: d40c->base->dev);
1457	spin_unlock_irqrestore(lock: &d40c->lock, flags);
1458	return res;
1459	}
1460
1461	static int d40_resume(struct dma_chan *chan)
1462	{
1463	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
1464	int res = 0;
1465	unsigned long flags;
1466
1467	if (d40c->phy_chan == NULL) {
1468	chan_err(d40c, "Channel is not allocated!\n");
1469	return -EINVAL;
1470	}
1471
1472	if (!d40c->busy)
1473	return 0;
1474
1475	spin_lock_irqsave(&d40c->lock, flags);
1476	pm_runtime_get_sync(dev: d40c->base->dev);
1477
1478	/* If bytes left to transfer or linked tx resume job */
1479	if (d40_residue(d40c) || d40_tx_is_linked(d40c))
1480	res = d40_channel_execute_command(d40c, command: D40_DMA_RUN);
1481
1482	pm_runtime_mark_last_busy(dev: d40c->base->dev);
1483	pm_runtime_put_autosuspend(dev: d40c->base->dev);
1484	spin_unlock_irqrestore(lock: &d40c->lock, flags);
1485	return res;
1486	}
1487
1488	static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1489	{
1490	struct d40_chan *d40c = container_of(tx->chan,
1491	struct d40_chan,
1492	chan);
1493	struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1494	unsigned long flags;
1495	dma_cookie_t cookie;
1496
1497	spin_lock_irqsave(&d40c->lock, flags);
1498	cookie = dma_cookie_assign(tx);
1499	d40_desc_queue(d40c, desc: d40d);
1500	spin_unlock_irqrestore(lock: &d40c->lock, flags);
1501
1502	return cookie;
1503	}
1504
1505	static int d40_start(struct d40_chan *d40c)
1506	{
1507	return d40_channel_execute_command(d40c, command: D40_DMA_RUN);
1508	}
1509
1510	static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1511	{
1512	struct d40_desc *d40d;
1513	int err;
1514
1515	/* Start queued jobs, if any */
1516	d40d = d40_first_queued(d40c);
1517
1518	if (d40d != NULL) {
1519	if (!d40c->busy) {
1520	d40c->busy = true;
1521	pm_runtime_get_sync(dev: d40c->base->dev);
1522	}
1523
1524	/* Remove from queue */
1525	d40_desc_remove(d40d);
1526
1527	/* Add to active queue */
1528	d40_desc_submit(d40c, desc: d40d);
1529
1530	/* Initiate DMA job */
1531	d40_desc_load(d40c, d40d);
1532
1533	/* Start dma job */
1534	err = d40_start(d40c);
1535
1536	if (err)
1537	return NULL;
1538	}
1539
1540	return d40d;
1541	}
1542
1543	/* called from interrupt context */
1544	static void dma_tc_handle(struct d40_chan *d40c)
1545	{
1546	struct d40_desc *d40d;
1547
1548	/* Get first active entry from list */
1549	d40d = d40_first_active_get(d40c);
1550
1551	if (d40d == NULL)
1552	return;
1553
1554	if (d40d->cyclic) {
1555	/*
1556	* If this was a paritially loaded list, we need to reloaded
1557	* it, and only when the list is completed. We need to check
1558	* for done because the interrupt will hit for every link, and
1559	* not just the last one.
1560	*/
1561	if (d40d->lli_current < d40d->lli_len
1562	&& !d40_tx_is_linked(d40c)
1563	&& !d40_residue(d40c)) {
1564	d40_lcla_free_all(d40c, d40d);
1565	d40_desc_load(d40c, d40d);
1566	(void) d40_start(d40c);
1567
1568	if (d40d->lli_current == d40d->lli_len)
1569	d40d->lli_current = 0;
1570	}
1571	} else {
1572	d40_lcla_free_all(d40c, d40d);
1573
1574	if (d40d->lli_current < d40d->lli_len) {
1575	d40_desc_load(d40c, d40d);
1576	/* Start dma job */
1577	(void) d40_start(d40c);
1578	return;
1579	}
1580
1581	if (d40_queue_start(d40c) == NULL) {
1582	d40c->busy = false;
1583
1584	pm_runtime_mark_last_busy(dev: d40c->base->dev);
1585	pm_runtime_put_autosuspend(dev: d40c->base->dev);
1586	}
1587
1588	d40_desc_remove(d40d);
1589	d40_desc_done(d40c, desc: d40d);
1590	}
1591
1592	d40c->pending_tx++;
1593	tasklet_schedule(t: &d40c->tasklet);
1594
1595	}
1596
1597	static void dma_tasklet(struct tasklet_struct *t)
1598	{
1599	struct d40_chan *d40c = from_tasklet(d40c, t, tasklet);
1600	struct d40_desc *d40d;
1601	unsigned long flags;
1602	bool callback_active;
1603	struct dmaengine_desc_callback cb;
1604
1605	spin_lock_irqsave(&d40c->lock, flags);
1606
1607	/* Get first entry from the done list */
1608	d40d = d40_first_done(d40c);
1609	if (d40d == NULL) {
1610	/* Check if we have reached here for cyclic job */
1611	d40d = d40_first_active_get(d40c);
1612	if (d40d == NULL || !d40d->cyclic)
1613	goto check_pending_tx;
1614	}
1615
1616	if (!d40d->cyclic)
1617	dma_cookie_complete(tx: &d40d->txd);
1618
1619	/*
1620	* If terminating a channel pending_tx is set to zero.
1621	* This prevents any finished active jobs to return to the client.
1622	*/
1623	if (d40c->pending_tx == 0) {
1624	spin_unlock_irqrestore(lock: &d40c->lock, flags);
1625	return;
1626	}
1627
1628	/* Callback to client */
1629	callback_active = !!(d40d->txd.flags & DMA_PREP_INTERRUPT);
1630	dmaengine_desc_get_callback(tx: &d40d->txd, cb: &cb);
1631
1632	if (!d40d->cyclic) {
1633	if (async_tx_test_ack(tx: &d40d->txd)) {
1634	d40_desc_remove(d40d);
1635	d40_desc_free(d40c, d40d);
1636	} else if (!d40d->is_in_client_list) {
1637	d40_desc_remove(d40d);
1638	d40_lcla_free_all(d40c, d40d);
1639	list_add_tail(new: &d40d->node, head: &d40c->client);
1640	d40d->is_in_client_list = true;
1641	}
1642	}
1643
1644	d40c->pending_tx--;
1645
1646	if (d40c->pending_tx)
1647	tasklet_schedule(t: &d40c->tasklet);
1648
1649	spin_unlock_irqrestore(lock: &d40c->lock, flags);
1650
1651	if (callback_active)
1652	dmaengine_desc_callback_invoke(cb: &cb, NULL);
1653
1654	return;
1655	check_pending_tx:
1656	/* Rescue maneuver if receiving double interrupts */
1657	if (d40c->pending_tx > 0)
1658	d40c->pending_tx--;
1659	spin_unlock_irqrestore(lock: &d40c->lock, flags);
1660	}
1661
1662	static irqreturn_t d40_handle_interrupt(int irq, void *data)
1663	{
1664	int i;
1665	u32 idx;
1666	u32 row;
1667	long chan = -1;
1668	struct d40_chan *d40c;
1669	struct d40_base *base = data;
1670	u32 *regs = base->regs_interrupt;
1671	struct d40_interrupt_lookup *il = base->gen_dmac.il;
1672	u32 il_size = base->gen_dmac.il_size;
1673
1674	spin_lock(lock: &base->interrupt_lock);
1675
1676	/* Read interrupt status of both logical and physical channels */
1677	for (i = 0; i < il_size; i++)
1678	regs[i] = readl(addr: base->virtbase + il[i].src);
1679
1680	for (;;) {
1681
1682	chan = find_next_bit(addr: (unsigned long *)regs,
1683	BITS_PER_LONG * il_size, offset: chan + 1);
1684
1685	/* No more set bits found? */
1686	if (chan == BITS_PER_LONG * il_size)
1687	break;
1688
1689	row = chan / BITS_PER_LONG;
1690	idx = chan & (BITS_PER_LONG - 1);
1691
1692	if (il[row].offset == D40_PHY_CHAN)
1693	d40c = base->lookup_phy_chans[idx];
1694	else
1695	d40c = base->lookup_log_chans[il[row].offset + idx];
1696
1697	if (!d40c) {
1698	/*
1699	* No error because this can happen if something else
1700	* in the system is using the channel.
1701	*/
1702	continue;
1703	}
1704
1705	/* ACK interrupt */
1706	writel(BIT(idx), addr: base->virtbase + il[row].clr);
1707
1708	spin_lock(lock: &d40c->lock);
1709
1710	if (!il[row].is_error)
1711	dma_tc_handle(d40c);
1712	else
1713	d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1714	chan, il[row].offset, idx);
1715
1716	spin_unlock(lock: &d40c->lock);
1717	}
1718
1719	spin_unlock(lock: &base->interrupt_lock);
1720
1721	return IRQ_HANDLED;
1722	}
1723
1724	static int d40_validate_conf(struct d40_chan *d40c,
1725	struct stedma40_chan_cfg *conf)
1726	{
1727	int res = 0;
1728	bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
1729
1730	if (!conf->dir) {
1731	chan_err(d40c, "Invalid direction.\n");
1732	res = -EINVAL;
1733	}
1734
1735	if ((is_log && conf->dev_type > d40c->base->num_log_chans) ||
1736	(!is_log && conf->dev_type > d40c->base->num_phy_chans) ||
1737	(conf->dev_type < 0)) {
1738	chan_err(d40c, "Invalid device type (%d)\n", conf->dev_type);
1739	res = -EINVAL;
1740	}
1741
1742	if (conf->dir == DMA_DEV_TO_DEV) {
1743	/*
1744	* DMAC HW supports it. Will be added to this driver,
1745	* in case any dma client requires it.
1746	*/
1747	chan_err(d40c, "periph to periph not supported\n");
1748	res = -EINVAL;
1749	}
1750
1751	if (d40_psize_2_burst_size(is_log, psize: conf->src_info.psize) *
1752	conf->src_info.data_width !=
1753	d40_psize_2_burst_size(is_log, psize: conf->dst_info.psize) *
1754	conf->dst_info.data_width) {
1755	/*
1756	* The DMAC hardware only supports
1757	* src (burst x width) == dst (burst x width)
1758	*/
1759
1760	chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
1761	res = -EINVAL;
1762	}
1763
1764	return res;
1765	}
1766
1767	static bool d40_alloc_mask_set(struct d40_phy_res *phy,
1768	bool is_src, int log_event_line, bool is_log,
1769	bool *first_user)
1770	{
1771	unsigned long flags;
1772	spin_lock_irqsave(&phy->lock, flags);
1773
1774	*first_user = ((phy->allocated_src | phy->allocated_dst)
1775	== D40_ALLOC_FREE);
1776
1777	if (!is_log) {
1778	/* Physical interrupts are masked per physical full channel */
1779	if (phy->allocated_src == D40_ALLOC_FREE &&
1780	phy->allocated_dst == D40_ALLOC_FREE) {
1781	phy->allocated_dst = D40_ALLOC_PHY;
1782	phy->allocated_src = D40_ALLOC_PHY;
1783	goto found_unlock;
1784	} else
1785	goto not_found_unlock;
1786	}
1787
1788	/* Logical channel */
1789	if (is_src) {
1790	if (phy->allocated_src == D40_ALLOC_PHY)
1791	goto not_found_unlock;
1792
1793	if (phy->allocated_src == D40_ALLOC_FREE)
1794	phy->allocated_src = D40_ALLOC_LOG_FREE;
1795
1796	if (!(phy->allocated_src & BIT(log_event_line))) {
1797	phy->allocated_src |= BIT(log_event_line);
1798	goto found_unlock;
1799	} else
1800	goto not_found_unlock;
1801	} else {
1802	if (phy->allocated_dst == D40_ALLOC_PHY)
1803	goto not_found_unlock;
1804
1805	if (phy->allocated_dst == D40_ALLOC_FREE)
1806	phy->allocated_dst = D40_ALLOC_LOG_FREE;
1807
1808	if (!(phy->allocated_dst & BIT(log_event_line))) {
1809	phy->allocated_dst |= BIT(log_event_line);
1810	goto found_unlock;
1811	}
1812	}
1813	not_found_unlock:
1814	spin_unlock_irqrestore(lock: &phy->lock, flags);
1815	return false;
1816	found_unlock:
1817	spin_unlock_irqrestore(lock: &phy->lock, flags);
1818	return true;
1819	}
1820
1821	static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1822	int log_event_line)
1823	{
1824	unsigned long flags;
1825	bool is_free = false;
1826
1827	spin_lock_irqsave(&phy->lock, flags);
1828	if (!log_event_line) {
1829	phy->allocated_dst = D40_ALLOC_FREE;
1830	phy->allocated_src = D40_ALLOC_FREE;
1831	is_free = true;
1832	goto unlock;
1833	}
1834
1835	/* Logical channel */
1836	if (is_src) {
1837	phy->allocated_src &= ~BIT(log_event_line);
1838	if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1839	phy->allocated_src = D40_ALLOC_FREE;
1840	} else {
1841	phy->allocated_dst &= ~BIT(log_event_line);
1842	if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1843	phy->allocated_dst = D40_ALLOC_FREE;
1844	}
1845
1846	is_free = ((phy->allocated_src | phy->allocated_dst) ==
1847	D40_ALLOC_FREE);
1848	unlock:
1849	spin_unlock_irqrestore(lock: &phy->lock, flags);
1850
1851	return is_free;
1852	}
1853
1854	static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user)
1855	{
1856	int dev_type = d40c->dma_cfg.dev_type;
1857	int event_group;
1858	int event_line;
1859	struct d40_phy_res *phys;
1860	int i;
1861	int j;
1862	int log_num;
1863	int num_phy_chans;
1864	bool is_src;
1865	bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
1866
1867	phys = d40c->base->phy_res;
1868	num_phy_chans = d40c->base->num_phy_chans;
1869
1870	if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) {
1871	log_num = 2 * dev_type;
1872	is_src = true;
1873	} else if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||
1874	d40c->dma_cfg.dir == DMA_MEM_TO_MEM) {
1875	/* dst event lines are used for logical memcpy */
1876	log_num = 2 * dev_type + 1;
1877	is_src = false;
1878	} else
1879	return -EINVAL;
1880
1881	event_group = D40_TYPE_TO_GROUP(dev_type);
1882	event_line = D40_TYPE_TO_EVENT(dev_type);
1883
1884	if (!is_log) {
1885	if (d40c->dma_cfg.dir == DMA_MEM_TO_MEM) {
1886	/* Find physical half channel */
1887	if (d40c->dma_cfg.use_fixed_channel) {
1888	i = d40c->dma_cfg.phy_channel;
1889	if (d40_alloc_mask_set(phy: &phys[i], is_src,
1890	log_event_line: 0, is_log,
1891	first_user: first_phy_user))
1892	goto found_phy;
1893	} else {
1894	for (i = 0; i < num_phy_chans; i++) {
1895	if (d40_alloc_mask_set(phy: &phys[i], is_src,
1896	log_event_line: 0, is_log,
1897	first_user: first_phy_user))
1898	goto found_phy;
1899	}
1900	}
1901	} else
1902	for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1903	int phy_num = j + event_group * 2;
1904	for (i = phy_num; i < phy_num + 2; i++) {
1905	if (d40_alloc_mask_set(phy: &phys[i],
1906	is_src,
1907	log_event_line: 0,
1908	is_log,
1909	first_user: first_phy_user))
1910	goto found_phy;
1911	}
1912	}
1913	return -EINVAL;
1914	found_phy:
1915	d40c->phy_chan = &phys[i];
1916	d40c->log_num = D40_PHY_CHAN;
1917	goto out;
1918	}
1919	if (dev_type == -1)
1920	return -EINVAL;
1921
1922	/* Find logical channel */
1923	for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1924	int phy_num = j + event_group * 2;
1925
1926	if (d40c->dma_cfg.use_fixed_channel) {
1927	i = d40c->dma_cfg.phy_channel;
1928
1929	if ((i != phy_num) && (i != phy_num + 1)) {
1930	dev_err(chan2dev(d40c),
1931	"invalid fixed phy channel %d\n", i);
1932	return -EINVAL;
1933	}
1934
1935	if (d40_alloc_mask_set(phy: &phys[i], is_src, log_event_line: event_line,
1936	is_log, first_user: first_phy_user))
1937	goto found_log;
1938
1939	dev_err(chan2dev(d40c),
1940	"could not allocate fixed phy channel %d\n", i);
1941	return -EINVAL;
1942	}
1943
1944	/*
1945	* Spread logical channels across all available physical rather
1946	* than pack every logical channel at the first available phy
1947	* channels.
1948	*/
1949	if (is_src) {
1950	for (i = phy_num; i < phy_num + 2; i++) {
1951	if (d40_alloc_mask_set(phy: &phys[i], is_src,
1952	log_event_line: event_line, is_log,
1953	first_user: first_phy_user))
1954	goto found_log;
1955	}
1956	} else {
1957	for (i = phy_num + 1; i >= phy_num; i--) {
1958	if (d40_alloc_mask_set(phy: &phys[i], is_src,
1959	log_event_line: event_line, is_log,
1960	first_user: first_phy_user))
1961	goto found_log;
1962	}
1963	}
1964	}
1965	return -EINVAL;
1966
1967	found_log:
1968	d40c->phy_chan = &phys[i];
1969	d40c->log_num = log_num;
1970	out:
1971
1972	if (is_log)
1973	d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1974	else
1975	d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1976
1977	return 0;
1978
1979	}
1980
1981	static int d40_config_memcpy(struct d40_chan *d40c)
1982	{
1983	dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1984
1985	if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1986	d40c->dma_cfg = dma40_memcpy_conf_log;
1987	d40c->dma_cfg.dev_type = dma40_memcpy_channels[d40c->chan.chan_id];
1988
1989	d40_log_cfg(cfg: &d40c->dma_cfg,
1990	lcsp1: &d40c->log_def.lcsp1, lcsp2: &d40c->log_def.lcsp3);
1991
1992	} else if (dma_has_cap(DMA_MEMCPY, cap) &&
1993	dma_has_cap(DMA_SLAVE, cap)) {
1994	d40c->dma_cfg = dma40_memcpy_conf_phy;
1995
1996	/* Generate interrupt at end of transfer or relink. */
1997	d40c->dst_def_cfg |= BIT(D40_SREG_CFG_TIM_POS);
1998
1999	/* Generate interrupt on error. */
2000	d40c->src_def_cfg |= BIT(D40_SREG_CFG_EIM_POS);
2001	d40c->dst_def_cfg |= BIT(D40_SREG_CFG_EIM_POS);
2002
2003	} else {
2004	chan_err(d40c, "No memcpy\n");
2005	return -EINVAL;
2006	}
2007
2008	return 0;
2009	}
2010
2011	static int d40_free_dma(struct d40_chan *d40c)
2012	{
2013
2014	int res = 0;
2015	u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
2016	struct d40_phy_res *phy = d40c->phy_chan;
2017	bool is_src;
2018
2019	/* Terminate all queued and active transfers */
2020	d40_term_all(d40c);
2021
2022	if (phy == NULL) {
2023	chan_err(d40c, "phy == null\n");
2024	return -EINVAL;
2025	}
2026
2027	if (phy->allocated_src == D40_ALLOC_FREE &&
2028	phy->allocated_dst == D40_ALLOC_FREE) {
2029	chan_err(d40c, "channel already free\n");
2030	return -EINVAL;
2031	}
2032
2033	if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||
2034	d40c->dma_cfg.dir == DMA_MEM_TO_MEM)
2035	is_src = false;
2036	else if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM)
2037	is_src = true;
2038	else {
2039	chan_err(d40c, "Unknown direction\n");
2040	return -EINVAL;
2041	}
2042
2043	pm_runtime_get_sync(dev: d40c->base->dev);
2044	res = d40_channel_execute_command(d40c, command: D40_DMA_STOP);
2045	if (res) {
2046	chan_err(d40c, "stop failed\n");
2047	goto mark_last_busy;
2048	}
2049
2050	d40_alloc_mask_free(phy, is_src, log_event_line: chan_is_logical(chan: d40c) ? event : 0);
2051
2052	if (chan_is_logical(chan: d40c))
2053	d40c->base->lookup_log_chans[d40c->log_num] = NULL;
2054	else
2055	d40c->base->lookup_phy_chans[phy->num] = NULL;
2056
2057	if (d40c->busy) {
2058	pm_runtime_mark_last_busy(dev: d40c->base->dev);
2059	pm_runtime_put_autosuspend(dev: d40c->base->dev);
2060	}
2061
2062	d40c->busy = false;
2063	d40c->phy_chan = NULL;
2064	d40c->configured = false;
2065	mark_last_busy:
2066	pm_runtime_mark_last_busy(dev: d40c->base->dev);
2067	pm_runtime_put_autosuspend(dev: d40c->base->dev);
2068	return res;
2069	}
2070
2071	static bool d40_is_paused(struct d40_chan *d40c)
2072	{
2073	void __iomem *chanbase = chan_base(chan: d40c);
2074	bool is_paused = false;
2075	unsigned long flags;
2076	void __iomem *active_reg;
2077	u32 status;
2078	u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type);
2079
2080	spin_lock_irqsave(&d40c->lock, flags);
2081
2082	if (chan_is_physical(chan: d40c)) {
2083	if (d40c->phy_chan->num % 2 == 0)
2084	active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
2085	else
2086	active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
2087
2088	status = (readl(addr: active_reg) &
2089	D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
2090	D40_CHAN_POS(d40c->phy_chan->num);
2091	if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
2092	is_paused = true;
2093	goto unlock;
2094	}
2095
2096	if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV ||
2097	d40c->dma_cfg.dir == DMA_MEM_TO_MEM) {
2098	status = readl(addr: chanbase + D40_CHAN_REG_SDLNK);
2099	} else if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) {
2100	status = readl(addr: chanbase + D40_CHAN_REG_SSLNK);
2101	} else {
2102	chan_err(d40c, "Unknown direction\n");
2103	goto unlock;
2104	}
2105
2106	status = (status & D40_EVENTLINE_MASK(event)) >>
2107	D40_EVENTLINE_POS(event);
2108
2109	if (status != D40_DMA_RUN)
2110	is_paused = true;
2111	unlock:
2112	spin_unlock_irqrestore(lock: &d40c->lock, flags);
2113	return is_paused;
2114
2115	}
2116
2117	static u32 stedma40_residue(struct dma_chan *chan)
2118	{
2119	struct d40_chan *d40c =
2120	container_of(chan, struct d40_chan, chan);
2121	u32 bytes_left;
2122	unsigned long flags;
2123
2124	spin_lock_irqsave(&d40c->lock, flags);
2125	bytes_left = d40_residue(d40c);
2126	spin_unlock_irqrestore(lock: &d40c->lock, flags);
2127
2128	return bytes_left;
2129	}
2130
2131	static int
2132	d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
2133	struct scatterlist *sg_src, struct scatterlist *sg_dst,
2134	unsigned int sg_len, dma_addr_t src_dev_addr,
2135	dma_addr_t dst_dev_addr)
2136	{
2137	struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2138	struct stedma40_half_channel_info *src_info = &cfg->src_info;
2139	struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
2140	int ret;
2141
2142	ret = d40_log_sg_to_lli(sg: sg_src, sg_len,
2143	dev_addr: src_dev_addr,
2144	lli_sg: desc->lli_log.src,
2145	lcsp13: chan->log_def.lcsp1,
2146	data_width1: src_info->data_width,
2147	data_width2: dst_info->data_width);
2148
2149	ret = d40_log_sg_to_lli(sg: sg_dst, sg_len,
2150	dev_addr: dst_dev_addr,
2151	lli_sg: desc->lli_log.dst,
2152	lcsp13: chan->log_def.lcsp3,
2153	data_width1: dst_info->data_width,
2154	data_width2: src_info->data_width);
2155
2156	return ret < 0 ? ret : 0;
2157	}
2158
2159	static int
2160	d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
2161	struct scatterlist *sg_src, struct scatterlist *sg_dst,
2162	unsigned int sg_len, dma_addr_t src_dev_addr,
2163	dma_addr_t dst_dev_addr)
2164	{
2165	struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
2166	struct stedma40_half_channel_info *src_info = &cfg->src_info;
2167	struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
2168	unsigned long flags = 0;
2169	int ret;
2170
2171	if (desc->cyclic)
2172	flags |= LLI_CYCLIC | LLI_TERM_INT;
2173
2174	ret = d40_phy_sg_to_lli(sg: sg_src, sg_len, target: src_dev_addr,
2175	lli: desc->lli_phy.src,
2176	virt_to_phys(address: desc->lli_phy.src),
2177	reg_cfg: chan->src_def_cfg,
2178	info: src_info, otherinfo: dst_info, flags);
2179
2180	ret = d40_phy_sg_to_lli(sg: sg_dst, sg_len, target: dst_dev_addr,
2181	lli: desc->lli_phy.dst,
2182	virt_to_phys(address: desc->lli_phy.dst),
2183	reg_cfg: chan->dst_def_cfg,
2184	info: dst_info, otherinfo: src_info, flags);
2185
2186	dma_sync_single_for_device(dev: chan->base->dev, addr: desc->lli_pool.dma_addr,
2187	size: desc->lli_pool.size, dir: DMA_TO_DEVICE);
2188
2189	return ret < 0 ? ret : 0;
2190	}
2191
2192	static struct d40_desc *
2193	d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
2194	unsigned int sg_len, unsigned long dma_flags)
2195	{
2196	struct stedma40_chan_cfg *cfg;
2197	struct d40_desc *desc;
2198	int ret;
2199
2200	desc = d40_desc_get(d40c: chan);
2201	if (!desc)
2202	return NULL;
2203
2204	cfg = &chan->dma_cfg;
2205	desc->lli_len = d40_sg_2_dmalen(sgl: sg, sg_len, data_width1: cfg->src_info.data_width,
2206	data_width2: cfg->dst_info.data_width);
2207	if (desc->lli_len < 0) {
2208	chan_err(chan, "Unaligned size\n");
2209	goto free_desc;
2210	}
2211
2212	ret = d40_pool_lli_alloc(d40c: chan, d40d: desc, lli_len: desc->lli_len);
2213	if (ret < 0) {
2214	chan_err(chan, "Could not allocate lli\n");
2215	goto free_desc;
2216	}
2217
2218	desc->lli_current = 0;
2219	desc->txd.flags = dma_flags;
2220	desc->txd.tx_submit = d40_tx_submit;
2221
2222	dma_async_tx_descriptor_init(tx: &desc->txd, chan: &chan->chan);
2223
2224	return desc;
2225	free_desc:
2226	d40_desc_free(d40c: chan, d40d: desc);
2227	return NULL;
2228	}
2229
2230	static struct dma_async_tx_descriptor *
2231	d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
2232	struct scatterlist *sg_dst, unsigned int sg_len,
2233	enum dma_transfer_direction direction, unsigned long dma_flags)
2234	{
2235	struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
2236	dma_addr_t src_dev_addr;
2237	dma_addr_t dst_dev_addr;
2238	struct d40_desc *desc;
2239	unsigned long flags;
2240	int ret;
2241
2242	if (!chan->phy_chan) {
2243	chan_err(chan, "Cannot prepare unallocated channel\n");
2244	return NULL;
2245	}
2246
2247	d40_set_runtime_config_write(chan: dchan, config: &chan->slave_config, direction);
2248
2249	spin_lock_irqsave(&chan->lock, flags);
2250
2251	desc = d40_prep_desc(chan, sg: sg_src, sg_len, dma_flags);
2252	if (desc == NULL)
2253	goto unlock;
2254
2255	if (sg_next(&sg_src[sg_len - 1]) == sg_src)
2256	desc->cyclic = true;
2257
2258	src_dev_addr = 0;
2259	dst_dev_addr = 0;
2260	if (direction == DMA_DEV_TO_MEM)
2261	src_dev_addr = chan->runtime_addr;
2262	else if (direction == DMA_MEM_TO_DEV)
2263	dst_dev_addr = chan->runtime_addr;
2264
2265	if (chan_is_logical(chan))
2266	ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
2267	sg_len, src_dev_addr, dst_dev_addr);
2268	else
2269	ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
2270	sg_len, src_dev_addr, dst_dev_addr);
2271
2272	if (ret) {
2273	chan_err(chan, "Failed to prepare %s sg job: %d\n",
2274	chan_is_logical(chan) ? "log" : "phy", ret);
2275	goto free_desc;
2276	}
2277
2278	/*
2279	* add descriptor to the prepare queue in order to be able
2280	* to free them later in terminate_all
2281	*/
2282	list_add_tail(new: &desc->node, head: &chan->prepare_queue);
2283
2284	spin_unlock_irqrestore(lock: &chan->lock, flags);
2285
2286	return &desc->txd;
2287	free_desc:
2288	d40_desc_free(d40c: chan, d40d: desc);
2289	unlock:
2290	spin_unlock_irqrestore(lock: &chan->lock, flags);
2291	return NULL;
2292	}
2293
2294	static bool stedma40_filter(struct dma_chan *chan, void *data)
2295	{
2296	struct stedma40_chan_cfg *info = data;
2297	struct d40_chan *d40c =
2298	container_of(chan, struct d40_chan, chan);
2299	int err;
2300
2301	if (data) {
2302	err = d40_validate_conf(d40c, conf: info);
2303	if (!err)
2304	d40c->dma_cfg = *info;
2305	} else
2306	err = d40_config_memcpy(d40c);
2307
2308	if (!err)
2309	d40c->configured = true;
2310
2311	return err == 0;
2312	}
2313
2314	static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
2315	{
2316	bool realtime = d40c->dma_cfg.realtime;
2317	bool highprio = d40c->dma_cfg.high_priority;
2318	u32 rtreg;
2319	u32 event = D40_TYPE_TO_EVENT(dev_type);
2320	u32 group = D40_TYPE_TO_GROUP(dev_type);
2321	u32 bit = BIT(event);
2322	u32 prioreg;
2323	struct d40_gen_dmac *dmac = &d40c->base->gen_dmac;
2324
2325	rtreg = realtime ? dmac->realtime_en : dmac->realtime_clear;
2326	/*
2327	* Due to a hardware bug, in some cases a logical channel triggered by
2328	* a high priority destination event line can generate extra packet
2329	* transactions.
2330	*
2331	* The workaround is to not set the high priority level for the
2332	* destination event lines that trigger logical channels.
2333	*/
2334	if (!src && chan_is_logical(chan: d40c))
2335	highprio = false;
2336
2337	prioreg = highprio ? dmac->high_prio_en : dmac->high_prio_clear;
2338
2339	/* Destination event lines are stored in the upper halfword */
2340	if (!src)
2341	bit <<= 16;
2342
2343	writel(val: bit, addr: d40c->base->virtbase + prioreg + group * 4);
2344	writel(val: bit, addr: d40c->base->virtbase + rtreg + group * 4);
2345	}
2346
2347	static void d40_set_prio_realtime(struct d40_chan *d40c)
2348	{
2349	if (d40c->base->rev < 3)
2350	return;
2351
2352	if ((d40c->dma_cfg.dir == DMA_DEV_TO_MEM) ||
2353	(d40c->dma_cfg.dir == DMA_DEV_TO_DEV))
2354	__d40_set_prio_rt(d40c, dev_type: d40c->dma_cfg.dev_type, src: true);
2355
2356	if ((d40c->dma_cfg.dir == DMA_MEM_TO_DEV) ||
2357	(d40c->dma_cfg.dir == DMA_DEV_TO_DEV))
2358	__d40_set_prio_rt(d40c, dev_type: d40c->dma_cfg.dev_type, src: false);
2359	}
2360
2361	#define D40_DT_FLAGS_MODE(flags) ((flags >> 0) & 0x1)
2362	#define D40_DT_FLAGS_DIR(flags) ((flags >> 1) & 0x1)
2363	#define D40_DT_FLAGS_BIG_ENDIAN(flags) ((flags >> 2) & 0x1)
2364	#define D40_DT_FLAGS_FIXED_CHAN(flags) ((flags >> 3) & 0x1)
2365	#define D40_DT_FLAGS_HIGH_PRIO(flags) ((flags >> 4) & 0x1)
2366
2367	static struct dma_chan *d40_xlate(struct of_phandle_args *dma_spec,
2368	struct of_dma *ofdma)
2369	{
2370	struct stedma40_chan_cfg cfg;
2371	dma_cap_mask_t cap;
2372	u32 flags;
2373
2374	memset(&cfg, 0, sizeof(struct stedma40_chan_cfg));
2375
2376	dma_cap_zero(cap);
2377	dma_cap_set(DMA_SLAVE, cap);
2378
2379	cfg.dev_type = dma_spec->args[0];
2380	flags = dma_spec->args[2];
2381
2382	switch (D40_DT_FLAGS_MODE(flags)) {
2383	case 0: cfg.mode = STEDMA40_MODE_LOGICAL; break;
2384	case 1: cfg.mode = STEDMA40_MODE_PHYSICAL; break;
2385	}
2386
2387	switch (D40_DT_FLAGS_DIR(flags)) {
2388	case 0:
2389	cfg.dir = DMA_MEM_TO_DEV;
2390	cfg.dst_info.big_endian = D40_DT_FLAGS_BIG_ENDIAN(flags);
2391	break;
2392	case 1:
2393	cfg.dir = DMA_DEV_TO_MEM;
2394	cfg.src_info.big_endian = D40_DT_FLAGS_BIG_ENDIAN(flags);
2395	break;
2396	}
2397
2398	if (D40_DT_FLAGS_FIXED_CHAN(flags)) {
2399	cfg.phy_channel = dma_spec->args[1];
2400	cfg.use_fixed_channel = true;
2401	}
2402
2403	if (D40_DT_FLAGS_HIGH_PRIO(flags))
2404	cfg.high_priority = true;
2405
2406	return dma_request_channel(cap, stedma40_filter, &cfg);
2407	}
2408
2409	/* DMA ENGINE functions */
2410	static int d40_alloc_chan_resources(struct dma_chan *chan)
2411	{
2412	int err;
2413	unsigned long flags;
2414	struct d40_chan *d40c =
2415	container_of(chan, struct d40_chan, chan);
2416	bool is_free_phy;
2417	spin_lock_irqsave(&d40c->lock, flags);
2418
2419	dma_cookie_init(chan);
2420
2421	/* If no dma configuration is set use default configuration (memcpy) */
2422	if (!d40c->configured) {
2423	err = d40_config_memcpy(d40c);
2424	if (err) {
2425	chan_err(d40c, "Failed to configure memcpy channel\n");
2426	goto mark_last_busy;
2427	}
2428	}
2429
2430	err = d40_allocate_channel(d40c, first_phy_user: &is_free_phy);
2431	if (err) {
2432	chan_err(d40c, "Failed to allocate channel\n");
2433	d40c->configured = false;
2434	goto mark_last_busy;
2435	}
2436
2437	pm_runtime_get_sync(dev: d40c->base->dev);
2438
2439	d40_set_prio_realtime(d40c);
2440
2441	if (chan_is_logical(chan: d40c)) {
2442	if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM)
2443	d40c->lcpa = d40c->base->lcpa_base +
2444	d40c->dma_cfg.dev_type * D40_LCPA_CHAN_SIZE;
2445	else
2446	d40c->lcpa = d40c->base->lcpa_base +
2447	d40c->dma_cfg.dev_type *
2448	D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
2449
2450	/* Unmask the Global Interrupt Mask. */
2451	d40c->src_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS);
2452	d40c->dst_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS);
2453	}
2454
2455	dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n",
2456	chan_is_logical(d40c) ? "logical" : "physical",
2457	d40c->phy_chan->num,
2458	d40c->dma_cfg.use_fixed_channel ? ", fixed" : "");
2459
2460
2461	/*
2462	* Only write channel configuration to the DMA if the physical
2463	* resource is free. In case of multiple logical channels
2464	* on the same physical resource, only the first write is necessary.
2465	*/
2466	if (is_free_phy)
2467	d40_config_write(d40c);
2468	mark_last_busy:
2469	pm_runtime_mark_last_busy(dev: d40c->base->dev);
2470	pm_runtime_put_autosuspend(dev: d40c->base->dev);
2471	spin_unlock_irqrestore(lock: &d40c->lock, flags);
2472	return err;
2473	}
2474
2475	static void d40_free_chan_resources(struct dma_chan *chan)
2476	{
2477	struct d40_chan *d40c =
2478	container_of(chan, struct d40_chan, chan);
2479	int err;
2480	unsigned long flags;
2481
2482	if (d40c->phy_chan == NULL) {
2483	chan_err(d40c, "Cannot free unallocated channel\n");
2484	return;
2485	}
2486
2487	spin_lock_irqsave(&d40c->lock, flags);
2488
2489	err = d40_free_dma(d40c);
2490
2491	if (err)
2492	chan_err(d40c, "Failed to free channel\n");
2493	spin_unlock_irqrestore(lock: &d40c->lock, flags);
2494	}
2495
2496	static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2497	dma_addr_t dst,
2498	dma_addr_t src,
2499	size_t size,
2500	unsigned long dma_flags)
2501	{
2502	struct scatterlist dst_sg;
2503	struct scatterlist src_sg;
2504
2505	sg_init_table(&dst_sg, 1);
2506	sg_init_table(&src_sg, 1);
2507
2508	sg_dma_address(&dst_sg) = dst;
2509	sg_dma_address(&src_sg) = src;
2510
2511	sg_dma_len(&dst_sg) = size;
2512	sg_dma_len(&src_sg) = size;
2513
2514	return d40_prep_sg(dchan: chan, sg_src: &src_sg, sg_dst: &dst_sg, sg_len: 1,
2515	direction: DMA_MEM_TO_MEM, dma_flags);
2516	}
2517
2518	static struct dma_async_tx_descriptor *
2519	d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2520	unsigned int sg_len, enum dma_transfer_direction direction,
2521	unsigned long dma_flags, void *context)
2522	{
2523	if (!is_slave_direction(direction))
2524	return NULL;
2525
2526	return d40_prep_sg(dchan: chan, sg_src: sgl, sg_dst: sgl, sg_len, direction, dma_flags);
2527	}
2528
2529	static struct dma_async_tx_descriptor *
2530	dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2531	size_t buf_len, size_t period_len,
2532	enum dma_transfer_direction direction, unsigned long flags)
2533	{
2534	unsigned int periods = buf_len / period_len;
2535	struct dma_async_tx_descriptor *txd;
2536	struct scatterlist *sg;
2537	int i;
2538
2539	sg = kcalloc(n: periods + 1, size: sizeof(struct scatterlist), GFP_NOWAIT);
2540	if (!sg)
2541	return NULL;
2542
2543	for (i = 0; i < periods; i++) {
2544	sg_dma_address(&sg[i]) = dma_addr;
2545	sg_dma_len(&sg[i]) = period_len;
2546	dma_addr += period_len;
2547	}
2548
2549	sg_chain(prv: sg, prv_nents: periods + 1, sgl: sg);
2550
2551	txd = d40_prep_sg(dchan: chan, sg_src: sg, sg_dst: sg, sg_len: periods, direction,
2552	dma_flags: DMA_PREP_INTERRUPT);
2553
2554	kfree(objp: sg);
2555
2556	return txd;
2557	}
2558
2559	static enum dma_status d40_tx_status(struct dma_chan *chan,
2560	dma_cookie_t cookie,
2561	struct dma_tx_state *txstate)
2562	{
2563	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2564	enum dma_status ret;
2565
2566	if (d40c->phy_chan == NULL) {
2567	chan_err(d40c, "Cannot read status of unallocated channel\n");
2568	return -EINVAL;
2569	}
2570
2571	ret = dma_cookie_status(chan, cookie, state: txstate);
2572	if (ret != DMA_COMPLETE && txstate)
2573	dma_set_residue(state: txstate, residue: stedma40_residue(chan));
2574
2575	if (d40_is_paused(d40c))
2576	ret = DMA_PAUSED;
2577
2578	return ret;
2579	}
2580
2581	static void d40_issue_pending(struct dma_chan *chan)
2582	{
2583	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2584	unsigned long flags;
2585
2586	if (d40c->phy_chan == NULL) {
2587	chan_err(d40c, "Channel is not allocated!\n");
2588	return;
2589	}
2590
2591	spin_lock_irqsave(&d40c->lock, flags);
2592
2593	list_splice_tail_init(list: &d40c->pending_queue, head: &d40c->queue);
2594
2595	/* Busy means that queued jobs are already being processed */
2596	if (!d40c->busy)
2597	(void) d40_queue_start(d40c);
2598
2599	spin_unlock_irqrestore(lock: &d40c->lock, flags);
2600	}
2601
2602	static int d40_terminate_all(struct dma_chan *chan)
2603	{
2604	unsigned long flags;
2605	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2606	int ret;
2607
2608	if (d40c->phy_chan == NULL) {
2609	chan_err(d40c, "Channel is not allocated!\n");
2610	return -EINVAL;
2611	}
2612
2613	spin_lock_irqsave(&d40c->lock, flags);
2614
2615	pm_runtime_get_sync(dev: d40c->base->dev);
2616	ret = d40_channel_execute_command(d40c, command: D40_DMA_STOP);
2617	if (ret)
2618	chan_err(d40c, "Failed to stop channel\n");
2619
2620	d40_term_all(d40c);
2621	pm_runtime_mark_last_busy(dev: d40c->base->dev);
2622	pm_runtime_put_autosuspend(dev: d40c->base->dev);
2623	if (d40c->busy) {
2624	pm_runtime_mark_last_busy(dev: d40c->base->dev);
2625	pm_runtime_put_autosuspend(dev: d40c->base->dev);
2626	}
2627	d40c->busy = false;
2628
2629	spin_unlock_irqrestore(lock: &d40c->lock, flags);
2630	return 0;
2631	}
2632
2633	static int
2634	dma40_config_to_halfchannel(struct d40_chan *d40c,
2635	struct stedma40_half_channel_info *info,
2636	u32 maxburst)
2637	{
2638	int psize;
2639
2640	if (chan_is_logical(chan: d40c)) {
2641	if (maxburst >= 16)
2642	psize = STEDMA40_PSIZE_LOG_16;
2643	else if (maxburst >= 8)
2644	psize = STEDMA40_PSIZE_LOG_8;
2645	else if (maxburst >= 4)
2646	psize = STEDMA40_PSIZE_LOG_4;
2647	else
2648	psize = STEDMA40_PSIZE_LOG_1;
2649	} else {
2650	if (maxburst >= 16)
2651	psize = STEDMA40_PSIZE_PHY_16;
2652	else if (maxburst >= 8)
2653	psize = STEDMA40_PSIZE_PHY_8;
2654	else if (maxburst >= 4)
2655	psize = STEDMA40_PSIZE_PHY_4;
2656	else
2657	psize = STEDMA40_PSIZE_PHY_1;
2658	}
2659
2660	info->psize = psize;
2661	info->flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2662
2663	return 0;
2664	}
2665
2666	static int d40_set_runtime_config(struct dma_chan *chan,
2667	struct dma_slave_config *config)
2668	{
2669	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2670
2671	memcpy(&d40c->slave_config, config, sizeof(*config));
2672
2673	return 0;
2674	}
2675
2676	/* Runtime reconfiguration extension */
2677	static int d40_set_runtime_config_write(struct dma_chan *chan,
2678	struct dma_slave_config *config,
2679	enum dma_transfer_direction direction)
2680	{
2681	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2682	struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2683	enum dma_slave_buswidth src_addr_width, dst_addr_width;
2684	dma_addr_t config_addr;
2685	u32 src_maxburst, dst_maxburst;
2686	int ret;
2687
2688	if (d40c->phy_chan == NULL) {
2689	chan_err(d40c, "Channel is not allocated!\n");
2690	return -EINVAL;
2691	}
2692
2693	src_addr_width = config->src_addr_width;
2694	src_maxburst = config->src_maxburst;
2695	dst_addr_width = config->dst_addr_width;
2696	dst_maxburst = config->dst_maxburst;
2697
2698	if (direction == DMA_DEV_TO_MEM) {
2699	config_addr = config->src_addr;
2700
2701	if (cfg->dir != DMA_DEV_TO_MEM)
2702	dev_dbg(d40c->base->dev,
2703	"channel was not configured for peripheral "
2704	"to memory transfer (%d) overriding\n",
2705	cfg->dir);
2706	cfg->dir = DMA_DEV_TO_MEM;
2707
2708	/* Configure the memory side */
2709	if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2710	dst_addr_width = src_addr_width;
2711	if (dst_maxburst == 0)
2712	dst_maxburst = src_maxburst;
2713
2714	} else if (direction == DMA_MEM_TO_DEV) {
2715	config_addr = config->dst_addr;
2716
2717	if (cfg->dir != DMA_MEM_TO_DEV)
2718	dev_dbg(d40c->base->dev,
2719	"channel was not configured for memory "
2720	"to peripheral transfer (%d) overriding\n",
2721	cfg->dir);
2722	cfg->dir = DMA_MEM_TO_DEV;
2723
2724	/* Configure the memory side */
2725	if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
2726	src_addr_width = dst_addr_width;
2727	if (src_maxburst == 0)
2728	src_maxburst = dst_maxburst;
2729	} else {
2730	dev_err(d40c->base->dev,
2731	"unrecognized channel direction %d\n",
2732	direction);
2733	return -EINVAL;
2734	}
2735
2736	if (config_addr <= 0) {
2737	dev_err(d40c->base->dev, "no address supplied\n");
2738	return -EINVAL;
2739	}
2740
2741	if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) {
2742	dev_err(d40c->base->dev,
2743	"src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2744	src_maxburst,
2745	src_addr_width,
2746	dst_maxburst,
2747	dst_addr_width);
2748	return -EINVAL;
2749	}
2750
2751	if (src_maxburst > 16) {
2752	src_maxburst = 16;
2753	dst_maxburst = src_maxburst * src_addr_width / dst_addr_width;
2754	} else if (dst_maxburst > 16) {
2755	dst_maxburst = 16;
2756	src_maxburst = dst_maxburst * dst_addr_width / src_addr_width;
2757	}
2758
2759	/* Only valid widths are; 1, 2, 4 and 8. */
2760	if (src_addr_width <= DMA_SLAVE_BUSWIDTH_UNDEFINED ||
2761	src_addr_width > DMA_SLAVE_BUSWIDTH_8_BYTES ||
2762	dst_addr_width <= DMA_SLAVE_BUSWIDTH_UNDEFINED ||
2763	dst_addr_width > DMA_SLAVE_BUSWIDTH_8_BYTES ||
2764	!is_power_of_2(n: src_addr_width) ||
2765	!is_power_of_2(n: dst_addr_width))
2766	return -EINVAL;
2767
2768	cfg->src_info.data_width = src_addr_width;
2769	cfg->dst_info.data_width = dst_addr_width;
2770
2771	ret = dma40_config_to_halfchannel(d40c, info: &cfg->src_info,
2772	maxburst: src_maxburst);
2773	if (ret)
2774	return ret;
2775
2776	ret = dma40_config_to_halfchannel(d40c, info: &cfg->dst_info,
2777	maxburst: dst_maxburst);
2778	if (ret)
2779	return ret;
2780
2781	/* Fill in register values */
2782	if (chan_is_logical(chan: d40c))
2783	d40_log_cfg(cfg, lcsp1: &d40c->log_def.lcsp1, lcsp2: &d40c->log_def.lcsp3);
2784	else
2785	d40_phy_cfg(cfg, src_cfg: &d40c->src_def_cfg, dst_cfg: &d40c->dst_def_cfg);
2786
2787	/* These settings will take precedence later */
2788	d40c->runtime_addr = config_addr;
2789	d40c->runtime_direction = direction;
2790	dev_dbg(d40c->base->dev,
2791	"configured channel %s for %s, data width %d/%d, "
2792	"maxburst %d/%d elements, LE, no flow control\n",
2793	dma_chan_name(chan),
2794	(direction == DMA_DEV_TO_MEM) ? "RX" : "TX",
2795	src_addr_width, dst_addr_width,
2796	src_maxburst, dst_maxburst);
2797
2798	return 0;
2799	}
2800
2801	/* Initialization functions */
2802
2803	static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2804	struct d40_chan *chans, int offset,
2805	int num_chans)
2806	{
2807	int i = 0;
2808	struct d40_chan *d40c;
2809
2810	INIT_LIST_HEAD(list: &dma->channels);
2811
2812	for (i = offset; i < offset + num_chans; i++) {
2813	d40c = &chans[i];
2814	d40c->base = base;
2815	d40c->chan.device = dma;
2816
2817	spin_lock_init(&d40c->lock);
2818
2819	d40c->log_num = D40_PHY_CHAN;
2820
2821	INIT_LIST_HEAD(list: &d40c->done);
2822	INIT_LIST_HEAD(list: &d40c->active);
2823	INIT_LIST_HEAD(list: &d40c->queue);
2824	INIT_LIST_HEAD(list: &d40c->pending_queue);
2825	INIT_LIST_HEAD(list: &d40c->client);
2826	INIT_LIST_HEAD(list: &d40c->prepare_queue);
2827
2828	tasklet_setup(t: &d40c->tasklet, callback: dma_tasklet);
2829
2830	list_add_tail(new: &d40c->chan.device_node,
2831	head: &dma->channels);
2832	}
2833	}
2834
2835	static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2836	{
2837	if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) {
2838	dev->device_prep_slave_sg = d40_prep_slave_sg;
2839	dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2840	}
2841
2842	if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2843	dev->device_prep_dma_memcpy = d40_prep_memcpy;
2844	dev->directions = BIT(DMA_MEM_TO_MEM);
2845	/*
2846	* This controller can only access address at even
2847	* 32bit boundaries, i.e. 2^2
2848	*/
2849	dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
2850	}
2851
2852	if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2853	dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2854
2855	dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2856	dev->device_free_chan_resources = d40_free_chan_resources;
2857	dev->device_issue_pending = d40_issue_pending;
2858	dev->device_tx_status = d40_tx_status;
2859	dev->device_config = d40_set_runtime_config;
2860	dev->device_pause = d40_pause;
2861	dev->device_resume = d40_resume;
2862	dev->device_terminate_all = d40_terminate_all;
2863	dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
2864	dev->dev = base->dev;
2865	}
2866
2867	static int __init d40_dmaengine_init(struct d40_base *base,
2868	int num_reserved_chans)
2869	{
2870	int err ;
2871
2872	d40_chan_init(base, dma: &base->dma_slave, chans: base->log_chans,
2873	offset: 0, num_chans: base->num_log_chans);
2874
2875	dma_cap_zero(base->dma_slave.cap_mask);
2876	dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
2877	dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2878
2879	d40_ops_init(base, dev: &base->dma_slave);
2880
2881	err = dmaenginem_async_device_register(device: &base->dma_slave);
2882
2883	if (err) {
2884	d40_err(base->dev, "Failed to register slave channels\n");
2885	goto exit;
2886	}
2887
2888	d40_chan_init(base, dma: &base->dma_memcpy, chans: base->log_chans,
2889	offset: base->num_log_chans, num_chans: base->num_memcpy_chans);
2890
2891	dma_cap_zero(base->dma_memcpy.cap_mask);
2892	dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
2893
2894	d40_ops_init(base, dev: &base->dma_memcpy);
2895
2896	err = dmaenginem_async_device_register(device: &base->dma_memcpy);
2897
2898	if (err) {
2899	d40_err(base->dev,
2900	"Failed to register memcpy only channels\n");
2901	goto exit;
2902	}
2903
2904	d40_chan_init(base, dma: &base->dma_both, chans: base->phy_chans,
2905	offset: 0, num_chans: num_reserved_chans);
2906
2907	dma_cap_zero(base->dma_both.cap_mask);
2908	dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2909	dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
2910	dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
2911
2912	d40_ops_init(base, dev: &base->dma_both);
2913	err = dmaenginem_async_device_register(device: &base->dma_both);
2914
2915	if (err) {
2916	d40_err(base->dev,
2917	"Failed to register logical and physical capable channels\n");
2918	goto exit;
2919	}
2920	return 0;
2921	exit:
2922	return err;
2923	}
2924
2925	/* Suspend resume functionality */
2926	#ifdef CONFIG_PM_SLEEP
2927	static int dma40_suspend(struct device *dev)
2928	{
2929	struct d40_base *base = dev_get_drvdata(dev);
2930	int ret;
2931
2932	ret = pm_runtime_force_suspend(dev);
2933	if (ret)
2934	return ret;
2935
2936	if (base->lcpa_regulator)
2937	ret = regulator_disable(regulator: base->lcpa_regulator);
2938	return ret;
2939	}
2940
2941	static int dma40_resume(struct device *dev)
2942	{
2943	struct d40_base *base = dev_get_drvdata(dev);
2944	int ret = 0;
2945
2946	if (base->lcpa_regulator) {
2947	ret = regulator_enable(regulator: base->lcpa_regulator);
2948	if (ret)
2949	return ret;
2950	}
2951
2952	return pm_runtime_force_resume(dev);
2953	}
2954	#endif
2955
2956	#ifdef CONFIG_PM
2957	static void dma40_backup(void __iomem *baseaddr, u32 *backup,
2958	u32 *regaddr, int num, bool save)
2959	{
2960	int i;
2961
2962	for (i = 0; i < num; i++) {
2963	void __iomem *addr = baseaddr + regaddr[i];
2964
2965	if (save)
2966	backup[i] = readl_relaxed(addr);
2967	else
2968	writel_relaxed(backup[i], addr);
2969	}
2970	}
2971
2972	static void d40_save_restore_registers(struct d40_base *base, bool save)
2973	{
2974	int i;
2975
2976	/* Save/Restore channel specific registers */
2977	for (i = 0; i < base->num_phy_chans; i++) {
2978	void __iomem *addr;
2979	int idx;
2980
2981	if (base->phy_res[i].reserved)
2982	continue;
2983
2984	addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA;
2985	idx = i * ARRAY_SIZE(d40_backup_regs_chan);
2986
2987	dma40_backup(baseaddr: addr, backup: &base->reg_val_backup_chan[idx],
2988	regaddr: d40_backup_regs_chan,
2989	ARRAY_SIZE(d40_backup_regs_chan),
2990	save);
2991	}
2992
2993	/* Save/Restore global registers */
2994	dma40_backup(baseaddr: base->virtbase, backup: base->reg_val_backup,
2995	regaddr: d40_backup_regs, ARRAY_SIZE(d40_backup_regs),
2996	save);
2997
2998	/* Save/Restore registers only existing on dma40 v3 and later */
2999	if (base->gen_dmac.backup)
3000	dma40_backup(baseaddr: base->virtbase, backup: base->reg_val_backup_v4,
3001	regaddr: base->gen_dmac.backup,
3002	num: base->gen_dmac.backup_size,
3003	save);
3004	}
3005
3006	static int dma40_runtime_suspend(struct device *dev)
3007	{
3008	struct d40_base *base = dev_get_drvdata(dev);
3009
3010	d40_save_restore_registers(base, save: true);
3011
3012	/* Don't disable/enable clocks for v1 due to HW bugs */
3013	if (base->rev != 1)
3014	writel_relaxed(base->gcc_pwr_off_mask,
3015	base->virtbase + D40_DREG_GCC);
3016
3017	return 0;
3018	}
3019
3020	static int dma40_runtime_resume(struct device *dev)
3021	{
3022	struct d40_base *base = dev_get_drvdata(dev);
3023
3024	d40_save_restore_registers(base, save: false);
3025
3026	writel_relaxed(D40_DREG_GCC_ENABLE_ALL,
3027	base->virtbase + D40_DREG_GCC);
3028	return 0;
3029	}
3030	#endif
3031
3032	static const struct dev_pm_ops dma40_pm_ops = {
3033	SET_LATE_SYSTEM_SLEEP_PM_OPS(dma40_suspend, dma40_resume)
3034	SET_RUNTIME_PM_OPS(dma40_runtime_suspend,
3035	dma40_runtime_resume,
3036	NULL)
3037	};
3038
3039	/* Initialization functions. */
3040
3041	static int __init d40_phy_res_init(struct d40_base *base)
3042	{
3043	int i;
3044	int num_phy_chans_avail = 0;
3045	u32 val[2];
3046	int odd_even_bit = -2;
3047	int gcc = D40_DREG_GCC_ENA;
3048
3049	val[0] = readl(addr: base->virtbase + D40_DREG_PRSME);
3050	val[1] = readl(addr: base->virtbase + D40_DREG_PRSMO);
3051
3052	for (i = 0; i < base->num_phy_chans; i++) {
3053	base->phy_res[i].num = i;
3054	odd_even_bit += 2 * ((i % 2) == 0);
3055	if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
3056	/* Mark security only channels as occupied */
3057	base->phy_res[i].allocated_src = D40_ALLOC_PHY;
3058	base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
3059	base->phy_res[i].reserved = true;
3060	gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3061	D40_DREG_GCC_SRC);
3062	gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i),
3063	D40_DREG_GCC_DST);
3064
3065
3066	} else {
3067	base->phy_res[i].allocated_src = D40_ALLOC_FREE;
3068	base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
3069	base->phy_res[i].reserved = false;
3070	num_phy_chans_avail++;
3071	}
3072	spin_lock_init(&base->phy_res[i].lock);
3073	}
3074
3075	/* Mark disabled channels as occupied */
3076	for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
3077	int chan = base->plat_data->disabled_channels[i];
3078
3079	base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
3080	base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
3081	base->phy_res[chan].reserved = true;
3082	gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3083	D40_DREG_GCC_SRC);
3084	gcc |= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan),
3085	D40_DREG_GCC_DST);
3086	num_phy_chans_avail--;
3087	}
3088
3089	/* Mark soft_lli channels */
3090	for (i = 0; i < base->plat_data->num_of_soft_lli_chans; i++) {
3091	int chan = base->plat_data->soft_lli_chans[i];
3092
3093	base->phy_res[chan].use_soft_lli = true;
3094	}
3095
3096	dev_info(base->dev, "%d of %d physical DMA channels available\n",
3097	num_phy_chans_avail, base->num_phy_chans);
3098
3099	/* Verify settings extended vs standard */
3100	val[0] = readl(addr: base->virtbase + D40_DREG_PRTYP);
3101
3102	for (i = 0; i < base->num_phy_chans; i++) {
3103
3104	if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
3105	(val[0] & 0x3) != 1)
3106	dev_info(base->dev,
3107	"[%s] INFO: channel %d is misconfigured (%d)\n",
3108	__func__, i, val[0] & 0x3);
3109
3110	val[0] = val[0] >> 2;
3111	}
3112
3113	/*
3114	* To keep things simple, Enable all clocks initially.
3115	* The clocks will get managed later post channel allocation.
3116	* The clocks for the event lines on which reserved channels exists
3117	* are not managed here.
3118	*/
3119	writel(D40_DREG_GCC_ENABLE_ALL, addr: base->virtbase + D40_DREG_GCC);
3120	base->gcc_pwr_off_mask = gcc;
3121
3122	return num_phy_chans_avail;
3123	}
3124
3125	/* Called from the registered devm action */
3126	static void d40_drop_kmem_cache_action(void *d)
3127	{
3128	struct kmem_cache *desc_slab = d;
3129
3130	kmem_cache_destroy(s: desc_slab);
3131	}
3132
3133	static int __init d40_hw_detect_init(struct platform_device *pdev,
3134	struct d40_base **retbase)
3135	{
3136	struct stedma40_platform_data *plat_data = dev_get_platdata(dev: &pdev->dev);
3137	struct device *dev = &pdev->dev;
3138	struct clk *clk;
3139	void __iomem *virtbase;
3140	struct d40_base *base;
3141	int num_log_chans;
3142	int num_phy_chans;
3143	int num_memcpy_chans;
3144	int i;
3145	u32 pid;
3146	u32 cid;
3147	u8 rev;
3148	int ret;
3149
3150	clk = devm_clk_get_enabled(dev, NULL);
3151	if (IS_ERR(ptr: clk))
3152	return PTR_ERR(ptr: clk);
3153
3154	/* Get IO for DMAC base address */
3155	virtbase = devm_platform_ioremap_resource_byname(pdev, name: "base");
3156	if (IS_ERR(ptr: virtbase))
3157	return PTR_ERR(ptr: virtbase);
3158
3159	/* This is just a regular AMBA PrimeCell ID actually */
3160	for (pid = 0, i = 0; i < 4; i++)
3161	pid |= (readl(addr: virtbase + SZ_4K - 0x20 + 4 * i)
3162	& 255) << (i * 8);
3163	for (cid = 0, i = 0; i < 4; i++)
3164	cid |= (readl(addr: virtbase + SZ_4K - 0x10 + 4 * i)
3165	& 255) << (i * 8);
3166
3167	if (cid != AMBA_CID) {
3168	d40_err(dev, "Unknown hardware! No PrimeCell ID\n");
3169	return -EINVAL;
3170	}
3171	if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) {
3172	d40_err(dev, "Unknown designer! Got %x wanted %x\n",
3173	AMBA_MANF_BITS(pid),
3174	AMBA_VENDOR_ST);
3175	return -EINVAL;
3176	}
3177	/*
3178	* HW revision:
3179	* DB8500ed has revision 0
3180	* ? has revision 1
3181	* DB8500v1 has revision 2
3182	* DB8500v2 has revision 3
3183	* AP9540v1 has revision 4
3184	* DB8540v1 has revision 4
3185	*/
3186	rev = AMBA_REV_BITS(pid);
3187	if (rev < 2) {
3188	d40_err(dev, "hardware revision: %d is not supported", rev);
3189	return -EINVAL;
3190	}
3191
3192	/* The number of physical channels on this HW */
3193	if (plat_data->num_of_phy_chans)
3194	num_phy_chans = plat_data->num_of_phy_chans;
3195	else
3196	num_phy_chans = 4 * (readl(addr: virtbase + D40_DREG_ICFG) & 0x7) + 4;
3197
3198	/* The number of channels used for memcpy */
3199	if (plat_data->num_of_memcpy_chans)
3200	num_memcpy_chans = plat_data->num_of_memcpy_chans;
3201	else
3202	num_memcpy_chans = ARRAY_SIZE(dma40_memcpy_channels);
3203
3204	num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY;
3205
3206	dev_info(dev,
3207	"hardware rev: %d with %d physical and %d logical channels\n",
3208	rev, num_phy_chans, num_log_chans);
3209
3210	base = devm_kzalloc(dev,
3211	ALIGN(sizeof(struct d40_base), 4) +
3212	(num_phy_chans + num_log_chans + num_memcpy_chans) *
3213	sizeof(struct d40_chan), GFP_KERNEL);
3214
3215	if (!base)
3216	return -ENOMEM;
3217
3218	base->rev = rev;
3219	base->clk = clk;
3220	base->num_memcpy_chans = num_memcpy_chans;
3221	base->num_phy_chans = num_phy_chans;
3222	base->num_log_chans = num_log_chans;
3223	base->virtbase = virtbase;
3224	base->plat_data = plat_data;
3225	base->dev = dev;
3226	base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
3227	base->log_chans = &base->phy_chans[num_phy_chans];
3228
3229	if (base->plat_data->num_of_phy_chans == 14) {
3230	base->gen_dmac.backup = d40_backup_regs_v4b;
3231	base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4B;
3232	base->gen_dmac.interrupt_en = D40_DREG_CPCMIS;
3233	base->gen_dmac.interrupt_clear = D40_DREG_CPCICR;
3234	base->gen_dmac.realtime_en = D40_DREG_CRSEG1;
3235	base->gen_dmac.realtime_clear = D40_DREG_CRCEG1;
3236	base->gen_dmac.high_prio_en = D40_DREG_CPSEG1;
3237	base->gen_dmac.high_prio_clear = D40_DREG_CPCEG1;
3238	base->gen_dmac.il = il_v4b;
3239	base->gen_dmac.il_size = ARRAY_SIZE(il_v4b);
3240	base->gen_dmac.init_reg = dma_init_reg_v4b;
3241	base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4b);
3242	} else {
3243	if (base->rev >= 3) {
3244	base->gen_dmac.backup = d40_backup_regs_v4a;
3245	base->gen_dmac.backup_size = BACKUP_REGS_SZ_V4A;
3246	}
3247	base->gen_dmac.interrupt_en = D40_DREG_PCMIS;
3248	base->gen_dmac.interrupt_clear = D40_DREG_PCICR;
3249	base->gen_dmac.realtime_en = D40_DREG_RSEG1;
3250	base->gen_dmac.realtime_clear = D40_DREG_RCEG1;
3251	base->gen_dmac.high_prio_en = D40_DREG_PSEG1;
3252	base->gen_dmac.high_prio_clear = D40_DREG_PCEG1;
3253	base->gen_dmac.il = il_v4a;
3254	base->gen_dmac.il_size = ARRAY_SIZE(il_v4a);
3255	base->gen_dmac.init_reg = dma_init_reg_v4a;
3256	base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a);
3257	}
3258
3259	base->phy_res = devm_kcalloc(dev, n: num_phy_chans,
3260	size: sizeof(*base->phy_res),
3261	GFP_KERNEL);
3262	if (!base->phy_res)
3263	return -ENOMEM;
3264
3265	base->lookup_phy_chans = devm_kcalloc(dev, n: num_phy_chans,
3266	size: sizeof(*base->lookup_phy_chans),
3267	GFP_KERNEL);
3268	if (!base->lookup_phy_chans)
3269	return -ENOMEM;
3270
3271	base->lookup_log_chans = devm_kcalloc(dev, n: num_log_chans,
3272	size: sizeof(*base->lookup_log_chans),
3273	GFP_KERNEL);
3274	if (!base->lookup_log_chans)
3275	return -ENOMEM;
3276
3277	base->reg_val_backup_chan = devm_kmalloc_array(dev, n: base->num_phy_chans,
3278	size: sizeof(d40_backup_regs_chan),
3279	GFP_KERNEL);
3280	if (!base->reg_val_backup_chan)
3281	return -ENOMEM;
3282
3283	base->lcla_pool.alloc_map = devm_kcalloc(dev, n: num_phy_chans
3284	* D40_LCLA_LINK_PER_EVENT_GRP,
3285	size: sizeof(*base->lcla_pool.alloc_map),
3286	GFP_KERNEL);
3287	if (!base->lcla_pool.alloc_map)
3288	return -ENOMEM;
3289
3290	base->regs_interrupt = devm_kmalloc_array(dev, n: base->gen_dmac.il_size,
3291	size: sizeof(*base->regs_interrupt),
3292	GFP_KERNEL);
3293	if (!base->regs_interrupt)
3294	return -ENOMEM;
3295
3296	base->desc_slab = kmem_cache_create(D40_NAME, size: sizeof(struct d40_desc),
3297	align: 0, SLAB_HWCACHE_ALIGN,
3298	NULL);
3299	if (!base->desc_slab)
3300	return -ENOMEM;
3301
3302	ret = devm_add_action_or_reset(dev, d40_drop_kmem_cache_action,
3303	base->desc_slab);
3304	if (ret)
3305	return ret;
3306
3307	*retbase = base;
3308
3309	return 0;
3310	}
3311
3312	static void __init d40_hw_init(struct d40_base *base)
3313	{
3314
3315	int i;
3316	u32 prmseo[2] = {0, 0};
3317	u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3318	u32 pcmis = 0;
3319	u32 pcicr = 0;
3320	struct d40_reg_val *dma_init_reg = base->gen_dmac.init_reg;
3321	u32 reg_size = base->gen_dmac.init_reg_size;
3322
3323	for (i = 0; i < reg_size; i++)
3324	writel(val: dma_init_reg[i].val,
3325	addr: base->virtbase + dma_init_reg[i].reg);
3326
3327	/* Configure all our dma channels to default settings */
3328	for (i = 0; i < base->num_phy_chans; i++) {
3329
3330	activeo[i % 2] = activeo[i % 2] << 2;
3331
3332	if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
3333	== D40_ALLOC_PHY) {
3334	activeo[i % 2] |= 3;
3335	continue;
3336	}
3337
3338	/* Enable interrupt # */
3339	pcmis = (pcmis << 1) | 1;
3340
3341	/* Clear interrupt # */
3342	pcicr = (pcicr << 1) | 1;
3343
3344	/* Set channel to physical mode */
3345	prmseo[i % 2] = prmseo[i % 2] << 2;
3346	prmseo[i % 2] |= 1;
3347
3348	}
3349
3350	writel(val: prmseo[1], addr: base->virtbase + D40_DREG_PRMSE);
3351	writel(val: prmseo[0], addr: base->virtbase + D40_DREG_PRMSO);
3352	writel(val: activeo[1], addr: base->virtbase + D40_DREG_ACTIVE);
3353	writel(val: activeo[0], addr: base->virtbase + D40_DREG_ACTIVO);
3354
3355	/* Write which interrupt to enable */
3356	writel(val: pcmis, addr: base->virtbase + base->gen_dmac.interrupt_en);
3357
3358	/* Write which interrupt to clear */
3359	writel(val: pcicr, addr: base->virtbase + base->gen_dmac.interrupt_clear);
3360
3361	/* These are __initdata and cannot be accessed after init */
3362	base->gen_dmac.init_reg = NULL;
3363	base->gen_dmac.init_reg_size = 0;
3364	}
3365
3366	static int __init d40_lcla_allocate(struct d40_base *base)
3367	{
3368	struct d40_lcla_pool *pool = &base->lcla_pool;
3369	unsigned long *page_list;
3370	int i, j;
3371	int ret;
3372
3373	/*
3374	* This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3375	* To full fill this hardware requirement without wasting 256 kb
3376	* we allocate pages until we get an aligned one.
3377	*/
3378	page_list = kmalloc_array(MAX_LCLA_ALLOC_ATTEMPTS,
3379	size: sizeof(*page_list),
3380	GFP_KERNEL);
3381	if (!page_list)
3382	return -ENOMEM;
3383
3384	/* Calculating how many pages that are required */
3385	base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
3386
3387	for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
3388	page_list[i] = __get_free_pages(GFP_KERNEL,
3389	order: base->lcla_pool.pages);
3390	if (!page_list[i]) {
3391
3392	d40_err(base->dev, "Failed to allocate %d pages.\n",
3393	base->lcla_pool.pages);
3394	ret = -ENOMEM;
3395
3396	for (j = 0; j < i; j++)
3397	free_pages(addr: page_list[j], order: base->lcla_pool.pages);
3398	goto free_page_list;
3399	}
3400
3401	if ((virt_to_phys(address: (void *)page_list[i]) &
3402	(LCLA_ALIGNMENT - 1)) == 0)
3403	break;
3404	}
3405
3406	for (j = 0; j < i; j++)
3407	free_pages(addr: page_list[j], order: base->lcla_pool.pages);
3408
3409	if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
3410	base->lcla_pool.base = (void *)page_list[i];
3411	} else {
3412	/*
3413	* After many attempts and no success with finding the correct
3414	* alignment, try with allocating a big buffer.
3415	*/
3416	dev_warn(base->dev,
3417	"[%s] Failed to get %d pages @ 18 bit align.\n",
3418	__func__, base->lcla_pool.pages);
3419	base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
3420	base->num_phy_chans +
3421	LCLA_ALIGNMENT,
3422	GFP_KERNEL);
3423	if (!base->lcla_pool.base_unaligned) {
3424	ret = -ENOMEM;
3425	goto free_page_list;
3426	}
3427
3428	base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
3429	LCLA_ALIGNMENT);
3430	}
3431
3432	pool->dma_addr = dma_map_single(base->dev, pool->base,
3433	SZ_1K * base->num_phy_chans,
3434	DMA_TO_DEVICE);
3435	if (dma_mapping_error(dev: base->dev, dma_addr: pool->dma_addr)) {
3436	pool->dma_addr = 0;
3437	ret = -ENOMEM;
3438	goto free_page_list;
3439	}
3440
3441	writel(virt_to_phys(address: base->lcla_pool.base),
3442	addr: base->virtbase + D40_DREG_LCLA);
3443	ret = 0;
3444	free_page_list:
3445	kfree(objp: page_list);
3446	return ret;
3447	}
3448
3449	static int __init d40_of_probe(struct device *dev,
3450	struct device_node *np)
3451	{
3452	struct stedma40_platform_data *pdata;
3453	int num_phy = 0, num_memcpy = 0, num_disabled = 0;
3454	const __be32 *list;
3455
3456	pdata = devm_kzalloc(dev, size: sizeof(*pdata), GFP_KERNEL);
3457	if (!pdata)
3458	return -ENOMEM;
3459
3460	/* If absent this value will be obtained from h/w. */
3461	of_property_read_u32(np, propname: "dma-channels", out_value: &num_phy);
3462	if (num_phy > 0)
3463	pdata->num_of_phy_chans = num_phy;
3464
3465	list = of_get_property(node: np, name: "memcpy-channels", lenp: &num_memcpy);
3466	num_memcpy /= sizeof(*list);
3467
3468	if (num_memcpy > D40_MEMCPY_MAX_CHANS || num_memcpy <= 0) {
3469	d40_err(dev,
3470	"Invalid number of memcpy channels specified (%d)\n",
3471	num_memcpy);
3472	return -EINVAL;
3473	}
3474	pdata->num_of_memcpy_chans = num_memcpy;
3475
3476	of_property_read_u32_array(np, propname: "memcpy-channels",
3477	out_values: dma40_memcpy_channels,
3478	sz: num_memcpy);
3479
3480	list = of_get_property(node: np, name: "disabled-channels", lenp: &num_disabled);
3481	num_disabled /= sizeof(*list);
3482
3483	if (num_disabled >= STEDMA40_MAX_PHYS || num_disabled < 0) {
3484	d40_err(dev,
3485	"Invalid number of disabled channels specified (%d)\n",
3486	num_disabled);
3487	return -EINVAL;
3488	}
3489
3490	of_property_read_u32_array(np, propname: "disabled-channels",
3491	out_values: pdata->disabled_channels,
3492	sz: num_disabled);
3493	pdata->disabled_channels[num_disabled] = -1;
3494
3495	dev->platform_data = pdata;
3496
3497	return 0;
3498	}
3499
3500	static int __init d40_probe(struct platform_device *pdev)
3501	{
3502	struct device *dev = &pdev->dev;
3503	struct device_node *np = pdev->dev.of_node;
3504	struct device_node *np_lcpa;
3505	struct d40_base *base;
3506	struct resource *res;
3507	struct resource res_lcpa;
3508	int num_reserved_chans;
3509	u32 val;
3510	int ret;
3511
3512	if (d40_of_probe(dev, np)) {
3513	ret = -ENOMEM;
3514	goto report_failure;
3515	}
3516
3517	ret = d40_hw_detect_init(pdev, retbase: &base);
3518	if (ret)
3519	goto report_failure;
3520
3521	num_reserved_chans = d40_phy_res_init(base);
3522
3523	platform_set_drvdata(pdev, data: base);
3524
3525	spin_lock_init(&base->interrupt_lock);
3526	spin_lock_init(&base->execmd_lock);
3527
3528	/* Get IO for logical channel parameter address (LCPA) */
3529	np_lcpa = of_parse_phandle(np, phandle_name: "sram", index: 0);
3530	if (!np_lcpa) {
3531	dev_err(dev, "no LCPA SRAM node\n");
3532	ret = -EINVAL;
3533	goto report_failure;
3534	}
3535	/* This is no device so read the address directly from the node */
3536	ret = of_address_to_resource(dev: np_lcpa, index: 0, r: &res_lcpa);
3537	if (ret) {
3538	dev_err(dev, "no LCPA SRAM resource\n");
3539	goto report_failure;
3540	}
3541	base->lcpa_size = resource_size(res: &res_lcpa);
3542	base->phy_lcpa = res_lcpa.start;
3543	dev_info(dev, "found LCPA SRAM at %pad, size %pa\n",
3544	&base->phy_lcpa, &base->lcpa_size);
3545
3546	/* We make use of ESRAM memory for this. */
3547	val = readl(addr: base->virtbase + D40_DREG_LCPA);
3548	if (base->phy_lcpa != val && val != 0) {
3549	dev_warn(dev,
3550	"[%s] Mismatch LCPA dma 0x%x, def %08x\n",
3551	__func__, val, (u32)base->phy_lcpa);
3552	} else
3553	writel(val: base->phy_lcpa, addr: base->virtbase + D40_DREG_LCPA);
3554
3555	base->lcpa_base = devm_ioremap(dev, offset: base->phy_lcpa, size: base->lcpa_size);
3556	if (!base->lcpa_base) {
3557	ret = -ENOMEM;
3558	d40_err(dev, "Failed to ioremap LCPA region\n");
3559	goto report_failure;
3560	}
3561	/* If lcla has to be located in ESRAM we don't need to allocate */
3562	if (base->plat_data->use_esram_lcla) {
3563	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
3564	"lcla_esram");
3565	if (!res) {
3566	ret = -ENOENT;
3567	d40_err(dev,
3568	"No \"lcla_esram\" memory resource\n");
3569	goto report_failure;
3570	}
3571	base->lcla_pool.base = devm_ioremap(dev, offset: res->start,
3572	size: resource_size(res));
3573	if (!base->lcla_pool.base) {
3574	ret = -ENOMEM;
3575	d40_err(dev, "Failed to ioremap LCLA region\n");
3576	goto report_failure;
3577	}
3578	writel(val: res->start, addr: base->virtbase + D40_DREG_LCLA);
3579
3580	} else {
3581	ret = d40_lcla_allocate(base);
3582	if (ret) {
3583	d40_err(dev, "Failed to allocate LCLA area\n");
3584	goto destroy_cache;
3585	}
3586	}
3587
3588	spin_lock_init(&base->lcla_pool.lock);
3589
3590	base->irq = platform_get_irq(pdev, 0);
3591	if (base->irq < 0) {
3592	ret = base->irq;
3593	goto destroy_cache;
3594	}
3595
3596	ret = request_irq(irq: base->irq, handler: d40_handle_interrupt, flags: 0, D40_NAME, dev: base);
3597	if (ret) {
3598	d40_err(dev, "No IRQ defined\n");
3599	goto destroy_cache;
3600	}
3601
3602	if (base->plat_data->use_esram_lcla) {
3603
3604	base->lcpa_regulator = regulator_get(dev: base->dev, id: "lcla_esram");
3605	if (IS_ERR(ptr: base->lcpa_regulator)) {
3606	d40_err(dev, "Failed to get lcpa_regulator\n");
3607	ret = PTR_ERR(ptr: base->lcpa_regulator);
3608	base->lcpa_regulator = NULL;
3609	goto destroy_cache;
3610	}
3611
3612	ret = regulator_enable(regulator: base->lcpa_regulator);
3613	if (ret) {
3614	d40_err(dev,
3615	"Failed to enable lcpa_regulator\n");
3616	regulator_put(regulator: base->lcpa_regulator);
3617	base->lcpa_regulator = NULL;
3618	goto destroy_cache;
3619	}
3620	}
3621
3622	writel_relaxed(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC);
3623
3624	pm_runtime_irq_safe(dev: base->dev);
3625	pm_runtime_set_autosuspend_delay(dev: base->dev, DMA40_AUTOSUSPEND_DELAY);
3626	pm_runtime_use_autosuspend(dev: base->dev);
3627	pm_runtime_mark_last_busy(dev: base->dev);
3628	pm_runtime_set_active(dev: base->dev);
3629	pm_runtime_enable(dev: base->dev);
3630
3631	ret = d40_dmaengine_init(base, num_reserved_chans);
3632	if (ret)
3633	goto destroy_cache;
3634
3635	ret = dma_set_max_seg_size(dev: base->dev, STEDMA40_MAX_SEG_SIZE);
3636	if (ret) {
3637	d40_err(dev, "Failed to set dma max seg size\n");
3638	goto destroy_cache;
3639	}
3640
3641	d40_hw_init(base);
3642
3643	ret = of_dma_controller_register(np, of_dma_xlate: d40_xlate, NULL);
3644	if (ret) {
3645	dev_err(dev,
3646	"could not register of_dma_controller\n");
3647	goto destroy_cache;
3648	}
3649
3650	dev_info(base->dev, "initialized\n");
3651	return 0;
3652
3653	destroy_cache:
3654	if (base->lcla_pool.dma_addr)
3655	dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
3656	SZ_1K * base->num_phy_chans,
3657	DMA_TO_DEVICE);
3658
3659	if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
3660	free_pages(addr: (unsigned long)base->lcla_pool.base,
3661	order: base->lcla_pool.pages);
3662
3663	kfree(objp: base->lcla_pool.base_unaligned);
3664
3665	if (base->lcpa_regulator) {
3666	regulator_disable(regulator: base->lcpa_regulator);
3667	regulator_put(regulator: base->lcpa_regulator);
3668	}
3669	pm_runtime_disable(dev: base->dev);
3670
3671	report_failure:
3672	d40_err(dev, "probe failed\n");
3673	return ret;
3674	}
3675
3676	static const struct of_device_id d40_match[] = {
3677	{ .compatible = "stericsson,dma40", },
3678	{}
3679	};
3680
3681	static struct platform_driver d40_driver = {
3682	.driver = {
3683	.name = D40_NAME,
3684	.pm = &dma40_pm_ops,
3685	.of_match_table = d40_match,
3686	},
3687	};
3688
3689	static int __init stedma40_init(void)
3690	{
3691	return platform_driver_probe(&d40_driver, d40_probe);
3692	}
3693	subsys_initcall(stedma40_init);
3694