1	// SPDX-License-Identifier: GPL-2.0+
2	// Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
3	// Copyright (C) 2008 Juergen Beisert
4
5	#include <linux/bits.h>
6	#include <linux/clk.h>
7	#include <linux/completion.h>
8	#include <linux/delay.h>
9	#include <linux/dmaengine.h>
10	#include <linux/dma-mapping.h>
11	#include <linux/err.h>
12	#include <linux/interrupt.h>
13	#include <linux/io.h>
14	#include <linux/irq.h>
15	#include <linux/kernel.h>
16	#include <linux/module.h>
17	#include <linux/pinctrl/consumer.h>
18	#include <linux/platform_device.h>
19	#include <linux/pm_runtime.h>
20	#include <linux/slab.h>
21	#include <linux/spi/spi.h>
22	#include <linux/types.h>
23	#include <linux/of.h>
24	#include <linux/property.h>
25
26	#include <linux/dma/imx-dma.h>
27
28	#define DRIVER_NAME "spi_imx"
29
30	static bool use_dma = true;
31	module_param(use_dma, bool, 0644);
32	MODULE_PARM_DESC(use_dma, "Enable usage of DMA when available (default)");
33
34	/* define polling limits */
35	static unsigned int polling_limit_us = 30;
36	module_param(polling_limit_us, uint, 0664);
37	MODULE_PARM_DESC(polling_limit_us,
38	"time in us to run a transfer in polling mode\n");
39
40	#define MXC_RPM_TIMEOUT 2000 /* 2000ms */
41
42	#define MXC_CSPIRXDATA 0x00
43	#define MXC_CSPITXDATA 0x04
44	#define MXC_CSPICTRL 0x08
45	#define MXC_CSPIINT 0x0c
46	#define MXC_RESET 0x1c
47
48	/* generic defines to abstract from the different register layouts */
49	#define MXC_INT_RR (1 << 0) /* Receive data ready interrupt */
50	#define MXC_INT_TE (1 << 1) /* Transmit FIFO empty interrupt */
51	#define MXC_INT_RDR BIT(4) /* Receive date threshold interrupt */
52
53	/* The maximum bytes that a sdma BD can transfer. */
54	#define MAX_SDMA_BD_BYTES (1 << 15)
55	#define MX51_ECSPI_CTRL_MAX_BURST 512
56	/* The maximum bytes that IMX53_ECSPI can transfer in target mode.*/
57	#define MX53_MAX_TRANSFER_BYTES 512
58
59	enum spi_imx_devtype {
60	IMX1_CSPI,
61	IMX21_CSPI,
62	IMX27_CSPI,
63	IMX31_CSPI,
64	IMX35_CSPI, /* CSPI on all i.mx except above */
65	IMX51_ECSPI, /* ECSPI on i.mx51 */
66	IMX53_ECSPI, /* ECSPI on i.mx53 and later */
67	};
68
69	struct spi_imx_data;
70
71	struct spi_imx_devtype_data {
72	void (*intctrl)(struct spi_imx_data *spi_imx, int enable);
73	int (*prepare_message)(struct spi_imx_data *spi_imx, struct spi_message *msg);
74	int (*prepare_transfer)(struct spi_imx_data *spi_imx, struct spi_device *spi);
75	void (*trigger)(struct spi_imx_data *spi_imx);
76	int (*rx_available)(struct spi_imx_data *spi_imx);
77	void (*reset)(struct spi_imx_data *spi_imx);
78	void (*setup_wml)(struct spi_imx_data *spi_imx);
79	void (*disable)(struct spi_imx_data *spi_imx);
80	bool has_dmamode;
81	bool has_targetmode;
82	unsigned int fifo_size;
83	bool dynamic_burst;
84	/*
85	* ERR009165 fixed or not:
86	* https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
87	*/
88	bool tx_glitch_fixed;
89	enum spi_imx_devtype devtype;
90	};
91
92	struct spi_imx_data {
93	struct spi_controller *controller;
94	struct device *dev;
95
96	struct completion xfer_done;
97	void __iomem *base;
98	unsigned long base_phys;
99
100	struct clk *clk_per;
101	struct clk *clk_ipg;
102	unsigned long spi_clk;
103	unsigned int spi_bus_clk;
104
105	unsigned int bits_per_word;
106	unsigned int spi_drctl;
107
108	unsigned int count, remainder;
109	void (*tx)(struct spi_imx_data *spi_imx);
110	void (*rx)(struct spi_imx_data *spi_imx);
111	void *rx_buf;
112	const void *tx_buf;
113	unsigned int txfifo; /* number of words pushed in tx FIFO */
114	unsigned int dynamic_burst;
115	bool rx_only;
116
117	/* Target mode */
118	bool target_mode;
119	bool target_aborted;
120	unsigned int target_burst;
121
122	/* DMA */
123	bool usedma;
124	u32 wml;
125	struct completion dma_rx_completion;
126	struct completion dma_tx_completion;
127
128	const struct spi_imx_devtype_data *devtype_data;
129	};
130
131	static inline int is_imx27_cspi(struct spi_imx_data *d)
132	{
133	return d->devtype_data->devtype == IMX27_CSPI;
134	}
135
136	static inline int is_imx35_cspi(struct spi_imx_data *d)
137	{
138	return d->devtype_data->devtype == IMX35_CSPI;
139	}
140
141	static inline int is_imx51_ecspi(struct spi_imx_data *d)
142	{
143	return d->devtype_data->devtype == IMX51_ECSPI;
144	}
145
146	static inline int is_imx53_ecspi(struct spi_imx_data *d)
147	{
148	return d->devtype_data->devtype == IMX53_ECSPI;
149	}
150
151	#define MXC_SPI_BUF_RX(type) \
152	static void spi_imx_buf_rx_##type(struct spi_imx_data *spi_imx) \
153	{ \
154	unsigned int val = readl(spi_imx->base + MXC_CSPIRXDATA); \
155	\
156	if (spi_imx->rx_buf) { \
157	*(type *)spi_imx->rx_buf = val; \
158	spi_imx->rx_buf += sizeof(type); \
159	} \
160	\
161	spi_imx->remainder -= sizeof(type); \
162	}
163
164	#define MXC_SPI_BUF_TX(type) \
165	static void spi_imx_buf_tx_##type(struct spi_imx_data *spi_imx) \
166	{ \
167	type val = 0; \
168	\
169	if (spi_imx->tx_buf) { \
170	val = *(type *)spi_imx->tx_buf; \
171	spi_imx->tx_buf += sizeof(type); \
172	} \
173	\
174	spi_imx->count -= sizeof(type); \
175	\
176	writel(val, spi_imx->base + MXC_CSPITXDATA); \
177	}
178
179	MXC_SPI_BUF_RX(u8)
180	MXC_SPI_BUF_TX(u8)
181	MXC_SPI_BUF_RX(u16)
182	MXC_SPI_BUF_TX(u16)
183	MXC_SPI_BUF_RX(u32)
184	MXC_SPI_BUF_TX(u32)
185
186	/* First entry is reserved, second entry is valid only if SDHC_SPIEN is set
187	* (which is currently not the case in this driver)
188	*/
189	static int mxc_clkdivs[] = {0, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192,
190	256, 384, 512, 768, 1024};
191
192	/* MX21, MX27 */
193	static unsigned int spi_imx_clkdiv_1(unsigned int fin,
194	unsigned int fspi, unsigned int max, unsigned int *fres)
195	{
196	int i;
197
198	for (i = 2; i < max; i++)
199	if (fspi * mxc_clkdivs[i] >= fin)
200	break;
201
202	*fres = fin / mxc_clkdivs[i];
203	return i;
204	}
205
206	/* MX1, MX31, MX35, MX51 CSPI */
207	static unsigned int spi_imx_clkdiv_2(unsigned int fin,
208	unsigned int fspi, unsigned int *fres)
209	{
210	int i, div = 4;
211
212	for (i = 0; i < 7; i++) {
213	if (fspi * div >= fin)
214	goto out;
215	div <<= 1;
216	}
217
218	out:
219	*fres = fin / div;
220	return i;
221	}
222
223	static int spi_imx_bytes_per_word(const int bits_per_word)
224	{
225	if (bits_per_word <= 8)
226	return 1;
227	else if (bits_per_word <= 16)
228	return 2;
229	else
230	return 4;
231	}
232
233	static bool spi_imx_can_dma(struct spi_controller *controller, struct spi_device *spi,
234	struct spi_transfer *transfer)
235	{
236	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: controller);
237
238	if (!use_dma || controller->fallback)
239	return false;
240
241	if (!controller->dma_rx)
242	return false;
243
244	if (spi_imx->target_mode)
245	return false;
246
247	if (transfer->len < spi_imx->devtype_data->fifo_size)
248	return false;
249
250	spi_imx->dynamic_burst = 0;
251
252	return true;
253	}
254
255	/*
256	* Note the number of natively supported chip selects for MX51 is 4. Some
257	* devices may have less actual SS pins but the register map supports 4. When
258	* using gpio chip selects the cs values passed into the macros below can go
259	* outside the range 0 - 3. We therefore need to limit the cs value to avoid
260	* corrupting bits outside the allocated locations.
261	*
262	* The simplest way to do this is to just mask the cs bits to 2 bits. This
263	* still allows all 4 native chip selects to work as well as gpio chip selects
264	* (which can use any of the 4 chip select configurations).
265	*/
266
267	#define MX51_ECSPI_CTRL 0x08
268	#define MX51_ECSPI_CTRL_ENABLE (1 << 0)
269	#define MX51_ECSPI_CTRL_XCH (1 << 2)
270	#define MX51_ECSPI_CTRL_SMC (1 << 3)
271	#define MX51_ECSPI_CTRL_MODE_MASK (0xf << 4)
272	#define MX51_ECSPI_CTRL_DRCTL(drctl) ((drctl) << 16)
273	#define MX51_ECSPI_CTRL_POSTDIV_OFFSET 8
274	#define MX51_ECSPI_CTRL_PREDIV_OFFSET 12
275	#define MX51_ECSPI_CTRL_CS(cs) ((cs & 3) << 18)
276	#define MX51_ECSPI_CTRL_BL_OFFSET 20
277	#define MX51_ECSPI_CTRL_BL_MASK (0xfff << 20)
278
279	#define MX51_ECSPI_CONFIG 0x0c
280	#define MX51_ECSPI_CONFIG_SCLKPHA(cs) (1 << ((cs & 3) + 0))
281	#define MX51_ECSPI_CONFIG_SCLKPOL(cs) (1 << ((cs & 3) + 4))
282	#define MX51_ECSPI_CONFIG_SBBCTRL(cs) (1 << ((cs & 3) + 8))
283	#define MX51_ECSPI_CONFIG_SSBPOL(cs) (1 << ((cs & 3) + 12))
284	#define MX51_ECSPI_CONFIG_DATACTL(cs) (1 << ((cs & 3) + 16))
285	#define MX51_ECSPI_CONFIG_SCLKCTL(cs) (1 << ((cs & 3) + 20))
286
287	#define MX51_ECSPI_INT 0x10
288	#define MX51_ECSPI_INT_TEEN (1 << 0)
289	#define MX51_ECSPI_INT_RREN (1 << 3)
290	#define MX51_ECSPI_INT_RDREN (1 << 4)
291
292	#define MX51_ECSPI_DMA 0x14
293	#define MX51_ECSPI_DMA_TX_WML(wml) ((wml) & 0x3f)
294	#define MX51_ECSPI_DMA_RX_WML(wml) (((wml) & 0x3f) << 16)
295	#define MX51_ECSPI_DMA_RXT_WML(wml) (((wml) & 0x3f) << 24)
296
297	#define MX51_ECSPI_DMA_TEDEN (1 << 7)
298	#define MX51_ECSPI_DMA_RXDEN (1 << 23)
299	#define MX51_ECSPI_DMA_RXTDEN (1 << 31)
300
301	#define MX51_ECSPI_STAT 0x18
302	#define MX51_ECSPI_STAT_RR (1 << 3)
303
304	#define MX51_ECSPI_TESTREG 0x20
305	#define MX51_ECSPI_TESTREG_LBC BIT(31)
306
307	static void spi_imx_buf_rx_swap_u32(struct spi_imx_data *spi_imx)
308	{
309	unsigned int val = readl(addr: spi_imx->base + MXC_CSPIRXDATA);
310
311	if (spi_imx->rx_buf) {
312	#ifdef __LITTLE_ENDIAN
313	unsigned int bytes_per_word;
314
315	bytes_per_word = spi_imx_bytes_per_word(bits_per_word: spi_imx->bits_per_word);
316	if (bytes_per_word == 1)
317	swab32s(p: &val);
318	else if (bytes_per_word == 2)
319	swahw32s(p: &val);
320	#endif
321	*(u32 *)spi_imx->rx_buf = val;
322	spi_imx->rx_buf += sizeof(u32);
323	}
324
325	spi_imx->remainder -= sizeof(u32);
326	}
327
328	static void spi_imx_buf_rx_swap(struct spi_imx_data *spi_imx)
329	{
330	int unaligned;
331	u32 val;
332
333	unaligned = spi_imx->remainder % 4;
334
335	if (!unaligned) {
336	spi_imx_buf_rx_swap_u32(spi_imx);
337	return;
338	}
339
340	if (spi_imx_bytes_per_word(bits_per_word: spi_imx->bits_per_word) == 2) {
341	spi_imx_buf_rx_u16(spi_imx);
342	return;
343	}
344
345	val = readl(addr: spi_imx->base + MXC_CSPIRXDATA);
346
347	while (unaligned--) {
348	if (spi_imx->rx_buf) {
349	*(u8 *)spi_imx->rx_buf = (val >> (8 * unaligned)) & 0xff;
350	spi_imx->rx_buf++;
351	}
352	spi_imx->remainder--;
353	}
354	}
355
356	static void spi_imx_buf_tx_swap_u32(struct spi_imx_data *spi_imx)
357	{
358	u32 val = 0;
359	#ifdef __LITTLE_ENDIAN
360	unsigned int bytes_per_word;
361	#endif
362
363	if (spi_imx->tx_buf) {
364	val = *(u32 *)spi_imx->tx_buf;
365	spi_imx->tx_buf += sizeof(u32);
366	}
367
368	spi_imx->count -= sizeof(u32);
369	#ifdef __LITTLE_ENDIAN
370	bytes_per_word = spi_imx_bytes_per_word(bits_per_word: spi_imx->bits_per_word);
371
372	if (bytes_per_word == 1)
373	swab32s(p: &val);
374	else if (bytes_per_word == 2)
375	swahw32s(p: &val);
376	#endif
377	writel(val, addr: spi_imx->base + MXC_CSPITXDATA);
378	}
379
380	static void spi_imx_buf_tx_swap(struct spi_imx_data *spi_imx)
381	{
382	int unaligned;
383	u32 val = 0;
384
385	unaligned = spi_imx->count % 4;
386
387	if (!unaligned) {
388	spi_imx_buf_tx_swap_u32(spi_imx);
389	return;
390	}
391
392	if (spi_imx_bytes_per_word(bits_per_word: spi_imx->bits_per_word) == 2) {
393	spi_imx_buf_tx_u16(spi_imx);
394	return;
395	}
396
397	while (unaligned--) {
398	if (spi_imx->tx_buf) {
399	val |= *(u8 *)spi_imx->tx_buf << (8 * unaligned);
400	spi_imx->tx_buf++;
401	}
402	spi_imx->count--;
403	}
404
405	writel(val, addr: spi_imx->base + MXC_CSPITXDATA);
406	}
407
408	static void mx53_ecspi_rx_target(struct spi_imx_data *spi_imx)
409	{
410	u32 val = be32_to_cpu(readl(spi_imx->base + MXC_CSPIRXDATA));
411
412	if (spi_imx->rx_buf) {
413	int n_bytes = spi_imx->target_burst % sizeof(val);
414
415	if (!n_bytes)
416	n_bytes = sizeof(val);
417
418	memcpy(spi_imx->rx_buf,
419	((u8 *)&val) + sizeof(val) - n_bytes, n_bytes);
420
421	spi_imx->rx_buf += n_bytes;
422	spi_imx->target_burst -= n_bytes;
423	}
424
425	spi_imx->remainder -= sizeof(u32);
426	}
427
428	static void mx53_ecspi_tx_target(struct spi_imx_data *spi_imx)
429	{
430	u32 val = 0;
431	int n_bytes = spi_imx->count % sizeof(val);
432
433	if (!n_bytes)
434	n_bytes = sizeof(val);
435
436	if (spi_imx->tx_buf) {
437	memcpy(((u8 *)&val) + sizeof(val) - n_bytes,
438	spi_imx->tx_buf, n_bytes);
439	val = cpu_to_be32(val);
440	spi_imx->tx_buf += n_bytes;
441	}
442
443	spi_imx->count -= n_bytes;
444
445	writel(val, addr: spi_imx->base + MXC_CSPITXDATA);
446	}
447
448	/* MX51 eCSPI */
449	static unsigned int mx51_ecspi_clkdiv(struct spi_imx_data *spi_imx,
450	unsigned int fspi, unsigned int *fres)
451	{
452	/*
453	* there are two 4-bit dividers, the pre-divider divides by
454	* $pre, the post-divider by 2^$post
455	*/
456	unsigned int pre, post;
457	unsigned int fin = spi_imx->spi_clk;
458
459	fspi = min(fspi, fin);
460
461	post = fls(x: fin) - fls(x: fspi);
462	if (fin > fspi << post)
463	post++;
464
465	/* now we have: (fin <= fspi << post) with post being minimal */
466
467	post = max(4U, post) - 4;
468	if (unlikely(post > 0xf)) {
469	dev_err(spi_imx->dev, "cannot set clock freq: %u (base freq: %u)\n",
470	fspi, fin);
471	return 0xff;
472	}
473
474	pre = DIV_ROUND_UP(fin, fspi << post) - 1;
475
476	dev_dbg(spi_imx->dev, "%s: fin: %u, fspi: %u, post: %u, pre: %u\n",
477	__func__, fin, fspi, post, pre);
478
479	/* Resulting frequency for the SCLK line. */
480	*fres = (fin / (pre + 1)) >> post;
481
482	return (pre << MX51_ECSPI_CTRL_PREDIV_OFFSET) |
483	(post << MX51_ECSPI_CTRL_POSTDIV_OFFSET);
484	}
485
486	static void mx51_ecspi_intctrl(struct spi_imx_data *spi_imx, int enable)
487	{
488	unsigned int val = 0;
489
490	if (enable & MXC_INT_TE)
491	val |= MX51_ECSPI_INT_TEEN;
492
493	if (enable & MXC_INT_RR)
494	val |= MX51_ECSPI_INT_RREN;
495
496	if (enable & MXC_INT_RDR)
497	val |= MX51_ECSPI_INT_RDREN;
498
499	writel(val, addr: spi_imx->base + MX51_ECSPI_INT);
500	}
501
502	static void mx51_ecspi_trigger(struct spi_imx_data *spi_imx)
503	{
504	u32 reg;
505
506	reg = readl(addr: spi_imx->base + MX51_ECSPI_CTRL);
507	reg |= MX51_ECSPI_CTRL_XCH;
508	writel(val: reg, addr: spi_imx->base + MX51_ECSPI_CTRL);
509	}
510
511	static void mx51_ecspi_disable(struct spi_imx_data *spi_imx)
512	{
513	u32 ctrl;
514
515	ctrl = readl(addr: spi_imx->base + MX51_ECSPI_CTRL);
516	ctrl &= ~MX51_ECSPI_CTRL_ENABLE;
517	writel(val: ctrl, addr: spi_imx->base + MX51_ECSPI_CTRL);
518	}
519
520	static int mx51_ecspi_channel(const struct spi_device *spi)
521	{
522	if (!spi_get_csgpiod(spi, idx: 0))
523	return spi_get_chipselect(spi, idx: 0);
524	return spi->controller->unused_native_cs;
525	}
526
527	static int mx51_ecspi_prepare_message(struct spi_imx_data *spi_imx,
528	struct spi_message *msg)
529	{
530	struct spi_device *spi = msg->spi;
531	struct spi_transfer *xfer;
532	u32 ctrl = MX51_ECSPI_CTRL_ENABLE;
533	u32 min_speed_hz = ~0U;
534	u32 testreg, delay;
535	u32 cfg = readl(addr: spi_imx->base + MX51_ECSPI_CONFIG);
536	u32 current_cfg = cfg;
537	int channel = mx51_ecspi_channel(spi);
538
539	/* set Host or Target mode */
540	if (spi_imx->target_mode)
541	ctrl &= ~MX51_ECSPI_CTRL_MODE_MASK;
542	else
543	ctrl |= MX51_ECSPI_CTRL_MODE_MASK;
544
545	/*
546	* Enable SPI_RDY handling (falling edge/level triggered).
547	*/
548	if (spi->mode & SPI_READY)
549	ctrl |= MX51_ECSPI_CTRL_DRCTL(spi_imx->spi_drctl);
550
551	/* set chip select to use */
552	ctrl |= MX51_ECSPI_CTRL_CS(channel);
553
554	/*
555	* The ctrl register must be written first, with the EN bit set other
556	* registers must not be written to.
557	*/
558	writel(val: ctrl, addr: spi_imx->base + MX51_ECSPI_CTRL);
559
560	testreg = readl(addr: spi_imx->base + MX51_ECSPI_TESTREG);
561	if (spi->mode & SPI_LOOP)
562	testreg |= MX51_ECSPI_TESTREG_LBC;
563	else
564	testreg &= ~MX51_ECSPI_TESTREG_LBC;
565	writel(val: testreg, addr: spi_imx->base + MX51_ECSPI_TESTREG);
566
567	/*
568	* eCSPI burst completion by Chip Select signal in Target mode
569	* is not functional for imx53 Soc, config SPI burst completed when
570	* BURST_LENGTH + 1 bits are received
571	*/
572	if (spi_imx->target_mode && is_imx53_ecspi(d: spi_imx))
573	cfg &= ~MX51_ECSPI_CONFIG_SBBCTRL(channel);
574	else
575	cfg |= MX51_ECSPI_CONFIG_SBBCTRL(channel);
576
577	if (spi->mode & SPI_CPOL) {
578	cfg |= MX51_ECSPI_CONFIG_SCLKPOL(channel);
579	cfg |= MX51_ECSPI_CONFIG_SCLKCTL(channel);
580	} else {
581	cfg &= ~MX51_ECSPI_CONFIG_SCLKPOL(channel);
582	cfg &= ~MX51_ECSPI_CONFIG_SCLKCTL(channel);
583	}
584
585	if (spi->mode & SPI_MOSI_IDLE_LOW)
586	cfg |= MX51_ECSPI_CONFIG_DATACTL(channel);
587	else
588	cfg &= ~MX51_ECSPI_CONFIG_DATACTL(channel);
589
590	if (spi->mode & SPI_CS_HIGH)
591	cfg |= MX51_ECSPI_CONFIG_SSBPOL(channel);
592	else
593	cfg &= ~MX51_ECSPI_CONFIG_SSBPOL(channel);
594
595	if (cfg == current_cfg)
596	return 0;
597
598	writel(val: cfg, addr: spi_imx->base + MX51_ECSPI_CONFIG);
599
600	/*
601	* Wait until the changes in the configuration register CONFIGREG
602	* propagate into the hardware. It takes exactly one tick of the
603	* SCLK clock, but we will wait two SCLK clock just to be sure. The
604	* effect of the delay it takes for the hardware to apply changes
605	* is noticable if the SCLK clock run very slow. In such a case, if
606	* the polarity of SCLK should be inverted, the GPIO ChipSelect might
607	* be asserted before the SCLK polarity changes, which would disrupt
608	* the SPI communication as the device on the other end would consider
609	* the change of SCLK polarity as a clock tick already.
610	*
611	* Because spi_imx->spi_bus_clk is only set in prepare_message
612	* callback, iterate over all the transfers in spi_message, find the
613	* one with lowest bus frequency, and use that bus frequency for the
614	* delay calculation. In case all transfers have speed_hz == 0, then
615	* min_speed_hz is ~0 and the resulting delay is zero.
616	*/
617	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
618	if (!xfer->speed_hz)
619	continue;
620	min_speed_hz = min(xfer->speed_hz, min_speed_hz);
621	}
622
623	delay = (2 * 1000000) / min_speed_hz;
624	if (likely(delay < 10)) /* SCLK is faster than 200 kHz */
625	udelay(delay);
626	else /* SCLK is _very_ slow */
627	usleep_range(min: delay, max: delay + 10);
628
629	return 0;
630	}
631
632	static void mx51_configure_cpha(struct spi_imx_data *spi_imx,
633	struct spi_device *spi)
634	{
635	bool cpha = (spi->mode & SPI_CPHA);
636	bool flip_cpha = (spi->mode & SPI_RX_CPHA_FLIP) && spi_imx->rx_only;
637	u32 cfg = readl(addr: spi_imx->base + MX51_ECSPI_CONFIG);
638	int channel = mx51_ecspi_channel(spi);
639
640	/* Flip cpha logical value iff flip_cpha */
641	cpha ^= flip_cpha;
642
643	if (cpha)
644	cfg |= MX51_ECSPI_CONFIG_SCLKPHA(channel);
645	else
646	cfg &= ~MX51_ECSPI_CONFIG_SCLKPHA(channel);
647
648	writel(val: cfg, addr: spi_imx->base + MX51_ECSPI_CONFIG);
649	}
650
651	static int mx51_ecspi_prepare_transfer(struct spi_imx_data *spi_imx,
652	struct spi_device *spi)
653	{
654	u32 ctrl = readl(addr: spi_imx->base + MX51_ECSPI_CTRL);
655	u32 clk;
656
657	/* Clear BL field and set the right value */
658	ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
659	if (spi_imx->target_mode && is_imx53_ecspi(d: spi_imx))
660	ctrl |= (spi_imx->target_burst * 8 - 1)
661	<< MX51_ECSPI_CTRL_BL_OFFSET;
662	else {
663	if (spi_imx->usedma) {
664	ctrl |= (spi_imx->bits_per_word - 1)
665	<< MX51_ECSPI_CTRL_BL_OFFSET;
666	} else {
667	if (spi_imx->count >= MX51_ECSPI_CTRL_MAX_BURST)
668	ctrl |= (MX51_ECSPI_CTRL_MAX_BURST * BITS_PER_BYTE - 1)
669	<< MX51_ECSPI_CTRL_BL_OFFSET;
670	else
671	ctrl |= (spi_imx->count / DIV_ROUND_UP(spi_imx->bits_per_word,
672	BITS_PER_BYTE) * spi_imx->bits_per_word - 1)
673	<< MX51_ECSPI_CTRL_BL_OFFSET;
674	}
675	}
676
677	/* set clock speed */
678	ctrl &= ~(0xf << MX51_ECSPI_CTRL_POSTDIV_OFFSET |
679	0xf << MX51_ECSPI_CTRL_PREDIV_OFFSET);
680	ctrl |= mx51_ecspi_clkdiv(spi_imx, fspi: spi_imx->spi_bus_clk, fres: &clk);
681	spi_imx->spi_bus_clk = clk;
682
683	mx51_configure_cpha(spi_imx, spi);
684
685	/*
686	* ERR009165: work in XHC mode instead of SMC as PIO on the chips
687	* before i.mx6ul.
688	*/
689	if (spi_imx->usedma && spi_imx->devtype_data->tx_glitch_fixed)
690	ctrl |= MX51_ECSPI_CTRL_SMC;
691	else
692	ctrl &= ~MX51_ECSPI_CTRL_SMC;
693
694	writel(val: ctrl, addr: spi_imx->base + MX51_ECSPI_CTRL);
695
696	return 0;
697	}
698
699	static void mx51_setup_wml(struct spi_imx_data *spi_imx)
700	{
701	u32 tx_wml = 0;
702
703	if (spi_imx->devtype_data->tx_glitch_fixed)
704	tx_wml = spi_imx->wml;
705	/*
706	* Configure the DMA register: setup the watermark
707	* and enable DMA request.
708	*/
709	writel(MX51_ECSPI_DMA_RX_WML(spi_imx->wml - 1) |
710	MX51_ECSPI_DMA_TX_WML(tx_wml) |
711	MX51_ECSPI_DMA_RXT_WML(spi_imx->wml) |
712	MX51_ECSPI_DMA_TEDEN | MX51_ECSPI_DMA_RXDEN |
713	MX51_ECSPI_DMA_RXTDEN, addr: spi_imx->base + MX51_ECSPI_DMA);
714	}
715
716	static int mx51_ecspi_rx_available(struct spi_imx_data *spi_imx)
717	{
718	return readl(addr: spi_imx->base + MX51_ECSPI_STAT) & MX51_ECSPI_STAT_RR;
719	}
720
721	static void mx51_ecspi_reset(struct spi_imx_data *spi_imx)
722	{
723	/* drain receive buffer */
724	while (mx51_ecspi_rx_available(spi_imx))
725	readl(addr: spi_imx->base + MXC_CSPIRXDATA);
726	}
727
728	#define MX31_INTREG_TEEN (1 << 0)
729	#define MX31_INTREG_RREN (1 << 3)
730
731	#define MX31_CSPICTRL_ENABLE (1 << 0)
732	#define MX31_CSPICTRL_HOST (1 << 1)
733	#define MX31_CSPICTRL_XCH (1 << 2)
734	#define MX31_CSPICTRL_SMC (1 << 3)
735	#define MX31_CSPICTRL_POL (1 << 4)
736	#define MX31_CSPICTRL_PHA (1 << 5)
737	#define MX31_CSPICTRL_SSCTL (1 << 6)
738	#define MX31_CSPICTRL_SSPOL (1 << 7)
739	#define MX31_CSPICTRL_BC_SHIFT 8
740	#define MX35_CSPICTRL_BL_SHIFT 20
741	#define MX31_CSPICTRL_CS_SHIFT 24
742	#define MX35_CSPICTRL_CS_SHIFT 12
743	#define MX31_CSPICTRL_DR_SHIFT 16
744
745	#define MX31_CSPI_DMAREG 0x10
746	#define MX31_DMAREG_RH_DEN (1<<4)
747	#define MX31_DMAREG_TH_DEN (1<<1)
748
749	#define MX31_CSPISTATUS 0x14
750	#define MX31_STATUS_RR (1 << 3)
751
752	#define MX31_CSPI_TESTREG 0x1C
753	#define MX31_TEST_LBC (1 << 14)
754
755	/* These functions also work for the i.MX35, but be aware that
756	* the i.MX35 has a slightly different register layout for bits
757	* we do not use here.
758	*/
759	static void mx31_intctrl(struct spi_imx_data *spi_imx, int enable)
760	{
761	unsigned int val = 0;
762
763	if (enable & MXC_INT_TE)
764	val |= MX31_INTREG_TEEN;
765	if (enable & MXC_INT_RR)
766	val |= MX31_INTREG_RREN;
767
768	writel(val, addr: spi_imx->base + MXC_CSPIINT);
769	}
770
771	static void mx31_trigger(struct spi_imx_data *spi_imx)
772	{
773	unsigned int reg;
774
775	reg = readl(addr: spi_imx->base + MXC_CSPICTRL);
776	reg |= MX31_CSPICTRL_XCH;
777	writel(val: reg, addr: spi_imx->base + MXC_CSPICTRL);
778	}
779
780	static int mx31_prepare_message(struct spi_imx_data *spi_imx,
781	struct spi_message *msg)
782	{
783	return 0;
784	}
785
786	static int mx31_prepare_transfer(struct spi_imx_data *spi_imx,
787	struct spi_device *spi)
788	{
789	unsigned int reg = MX31_CSPICTRL_ENABLE | MX31_CSPICTRL_HOST;
790	unsigned int clk;
791
792	reg |= spi_imx_clkdiv_2(fin: spi_imx->spi_clk, fspi: spi_imx->spi_bus_clk, fres: &clk) <<
793	MX31_CSPICTRL_DR_SHIFT;
794	spi_imx->spi_bus_clk = clk;
795
796	if (is_imx35_cspi(d: spi_imx)) {
797	reg |= (spi_imx->bits_per_word - 1) << MX35_CSPICTRL_BL_SHIFT;
798	reg |= MX31_CSPICTRL_SSCTL;
799	} else {
800	reg |= (spi_imx->bits_per_word - 1) << MX31_CSPICTRL_BC_SHIFT;
801	}
802
803	if (spi->mode & SPI_CPHA)
804	reg |= MX31_CSPICTRL_PHA;
805	if (spi->mode & SPI_CPOL)
806	reg |= MX31_CSPICTRL_POL;
807	if (spi->mode & SPI_CS_HIGH)
808	reg |= MX31_CSPICTRL_SSPOL;
809	if (!spi_get_csgpiod(spi, idx: 0))
810	reg |= (spi_get_chipselect(spi, idx: 0)) <<
811	(is_imx35_cspi(d: spi_imx) ? MX35_CSPICTRL_CS_SHIFT :
812	MX31_CSPICTRL_CS_SHIFT);
813
814	if (spi_imx->usedma)
815	reg |= MX31_CSPICTRL_SMC;
816
817	writel(val: reg, addr: spi_imx->base + MXC_CSPICTRL);
818
819	reg = readl(addr: spi_imx->base + MX31_CSPI_TESTREG);
820	if (spi->mode & SPI_LOOP)
821	reg |= MX31_TEST_LBC;
822	else
823	reg &= ~MX31_TEST_LBC;
824	writel(val: reg, addr: spi_imx->base + MX31_CSPI_TESTREG);
825
826	if (spi_imx->usedma) {
827	/*
828	* configure DMA requests when RXFIFO is half full and
829	* when TXFIFO is half empty
830	*/
831	writel(MX31_DMAREG_RH_DEN | MX31_DMAREG_TH_DEN,
832	addr: spi_imx->base + MX31_CSPI_DMAREG);
833	}
834
835	return 0;
836	}
837
838	static int mx31_rx_available(struct spi_imx_data *spi_imx)
839	{
840	return readl(addr: spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR;
841	}
842
843	static void mx31_reset(struct spi_imx_data *spi_imx)
844	{
845	/* drain receive buffer */
846	while (readl(addr: spi_imx->base + MX31_CSPISTATUS) & MX31_STATUS_RR)
847	readl(addr: spi_imx->base + MXC_CSPIRXDATA);
848	}
849
850	#define MX21_INTREG_RR (1 << 4)
851	#define MX21_INTREG_TEEN (1 << 9)
852	#define MX21_INTREG_RREN (1 << 13)
853
854	#define MX21_CSPICTRL_POL (1 << 5)
855	#define MX21_CSPICTRL_PHA (1 << 6)
856	#define MX21_CSPICTRL_SSPOL (1 << 8)
857	#define MX21_CSPICTRL_XCH (1 << 9)
858	#define MX21_CSPICTRL_ENABLE (1 << 10)
859	#define MX21_CSPICTRL_HOST (1 << 11)
860	#define MX21_CSPICTRL_DR_SHIFT 14
861	#define MX21_CSPICTRL_CS_SHIFT 19
862
863	static void mx21_intctrl(struct spi_imx_data *spi_imx, int enable)
864	{
865	unsigned int val = 0;
866
867	if (enable & MXC_INT_TE)
868	val |= MX21_INTREG_TEEN;
869	if (enable & MXC_INT_RR)
870	val |= MX21_INTREG_RREN;
871
872	writel(val, addr: spi_imx->base + MXC_CSPIINT);
873	}
874
875	static void mx21_trigger(struct spi_imx_data *spi_imx)
876	{
877	unsigned int reg;
878
879	reg = readl(addr: spi_imx->base + MXC_CSPICTRL);
880	reg |= MX21_CSPICTRL_XCH;
881	writel(val: reg, addr: spi_imx->base + MXC_CSPICTRL);
882	}
883
884	static int mx21_prepare_message(struct spi_imx_data *spi_imx,
885	struct spi_message *msg)
886	{
887	return 0;
888	}
889
890	static int mx21_prepare_transfer(struct spi_imx_data *spi_imx,
891	struct spi_device *spi)
892	{
893	unsigned int reg = MX21_CSPICTRL_ENABLE | MX21_CSPICTRL_HOST;
894	unsigned int max = is_imx27_cspi(d: spi_imx) ? 16 : 18;
895	unsigned int clk;
896
897	reg |= spi_imx_clkdiv_1(fin: spi_imx->spi_clk, fspi: spi_imx->spi_bus_clk, max, fres: &clk)
898	<< MX21_CSPICTRL_DR_SHIFT;
899	spi_imx->spi_bus_clk = clk;
900
901	reg |= spi_imx->bits_per_word - 1;
902
903	if (spi->mode & SPI_CPHA)
904	reg |= MX21_CSPICTRL_PHA;
905	if (spi->mode & SPI_CPOL)
906	reg |= MX21_CSPICTRL_POL;
907	if (spi->mode & SPI_CS_HIGH)
908	reg |= MX21_CSPICTRL_SSPOL;
909	if (!spi_get_csgpiod(spi, idx: 0))
910	reg |= spi_get_chipselect(spi, idx: 0) << MX21_CSPICTRL_CS_SHIFT;
911
912	writel(val: reg, addr: spi_imx->base + MXC_CSPICTRL);
913
914	return 0;
915	}
916
917	static int mx21_rx_available(struct spi_imx_data *spi_imx)
918	{
919	return readl(addr: spi_imx->base + MXC_CSPIINT) & MX21_INTREG_RR;
920	}
921
922	static void mx21_reset(struct spi_imx_data *spi_imx)
923	{
924	writel(val: 1, addr: spi_imx->base + MXC_RESET);
925	}
926
927	#define MX1_INTREG_RR (1 << 3)
928	#define MX1_INTREG_TEEN (1 << 8)
929	#define MX1_INTREG_RREN (1 << 11)
930
931	#define MX1_CSPICTRL_POL (1 << 4)
932	#define MX1_CSPICTRL_PHA (1 << 5)
933	#define MX1_CSPICTRL_XCH (1 << 8)
934	#define MX1_CSPICTRL_ENABLE (1 << 9)
935	#define MX1_CSPICTRL_HOST (1 << 10)
936	#define MX1_CSPICTRL_DR_SHIFT 13
937
938	static void mx1_intctrl(struct spi_imx_data *spi_imx, int enable)
939	{
940	unsigned int val = 0;
941
942	if (enable & MXC_INT_TE)
943	val |= MX1_INTREG_TEEN;
944	if (enable & MXC_INT_RR)
945	val |= MX1_INTREG_RREN;
946
947	writel(val, addr: spi_imx->base + MXC_CSPIINT);
948	}
949
950	static void mx1_trigger(struct spi_imx_data *spi_imx)
951	{
952	unsigned int reg;
953
954	reg = readl(addr: spi_imx->base + MXC_CSPICTRL);
955	reg |= MX1_CSPICTRL_XCH;
956	writel(val: reg, addr: spi_imx->base + MXC_CSPICTRL);
957	}
958
959	static int mx1_prepare_message(struct spi_imx_data *spi_imx,
960	struct spi_message *msg)
961	{
962	return 0;
963	}
964
965	static int mx1_prepare_transfer(struct spi_imx_data *spi_imx,
966	struct spi_device *spi)
967	{
968	unsigned int reg = MX1_CSPICTRL_ENABLE | MX1_CSPICTRL_HOST;
969	unsigned int clk;
970
971	reg |= spi_imx_clkdiv_2(fin: spi_imx->spi_clk, fspi: spi_imx->spi_bus_clk, fres: &clk) <<
972	MX1_CSPICTRL_DR_SHIFT;
973	spi_imx->spi_bus_clk = clk;
974
975	reg |= spi_imx->bits_per_word - 1;
976
977	if (spi->mode & SPI_CPHA)
978	reg |= MX1_CSPICTRL_PHA;
979	if (spi->mode & SPI_CPOL)
980	reg |= MX1_CSPICTRL_POL;
981
982	writel(val: reg, addr: spi_imx->base + MXC_CSPICTRL);
983
984	return 0;
985	}
986
987	static int mx1_rx_available(struct spi_imx_data *spi_imx)
988	{
989	return readl(addr: spi_imx->base + MXC_CSPIINT) & MX1_INTREG_RR;
990	}
991
992	static void mx1_reset(struct spi_imx_data *spi_imx)
993	{
994	writel(val: 1, addr: spi_imx->base + MXC_RESET);
995	}
996
997	static struct spi_imx_devtype_data imx1_cspi_devtype_data = {
998	.intctrl = mx1_intctrl,
999	.prepare_message = mx1_prepare_message,
1000	.prepare_transfer = mx1_prepare_transfer,
1001	.trigger = mx1_trigger,
1002	.rx_available = mx1_rx_available,
1003	.reset = mx1_reset,
1004	.fifo_size = 8,
1005	.has_dmamode = false,
1006	.dynamic_burst = false,
1007	.has_targetmode = false,
1008	.devtype = IMX1_CSPI,
1009	};
1010
1011	static struct spi_imx_devtype_data imx21_cspi_devtype_data = {
1012	.intctrl = mx21_intctrl,
1013	.prepare_message = mx21_prepare_message,
1014	.prepare_transfer = mx21_prepare_transfer,
1015	.trigger = mx21_trigger,
1016	.rx_available = mx21_rx_available,
1017	.reset = mx21_reset,
1018	.fifo_size = 8,
1019	.has_dmamode = false,
1020	.dynamic_burst = false,
1021	.has_targetmode = false,
1022	.devtype = IMX21_CSPI,
1023	};
1024
1025	static struct spi_imx_devtype_data imx27_cspi_devtype_data = {
1026	/* i.mx27 cspi shares the functions with i.mx21 one */
1027	.intctrl = mx21_intctrl,
1028	.prepare_message = mx21_prepare_message,
1029	.prepare_transfer = mx21_prepare_transfer,
1030	.trigger = mx21_trigger,
1031	.rx_available = mx21_rx_available,
1032	.reset = mx21_reset,
1033	.fifo_size = 8,
1034	.has_dmamode = false,
1035	.dynamic_burst = false,
1036	.has_targetmode = false,
1037	.devtype = IMX27_CSPI,
1038	};
1039
1040	static struct spi_imx_devtype_data imx31_cspi_devtype_data = {
1041	.intctrl = mx31_intctrl,
1042	.prepare_message = mx31_prepare_message,
1043	.prepare_transfer = mx31_prepare_transfer,
1044	.trigger = mx31_trigger,
1045	.rx_available = mx31_rx_available,
1046	.reset = mx31_reset,
1047	.fifo_size = 8,
1048	.has_dmamode = false,
1049	.dynamic_burst = false,
1050	.has_targetmode = false,
1051	.devtype = IMX31_CSPI,
1052	};
1053
1054	static struct spi_imx_devtype_data imx35_cspi_devtype_data = {
1055	/* i.mx35 and later cspi shares the functions with i.mx31 one */
1056	.intctrl = mx31_intctrl,
1057	.prepare_message = mx31_prepare_message,
1058	.prepare_transfer = mx31_prepare_transfer,
1059	.trigger = mx31_trigger,
1060	.rx_available = mx31_rx_available,
1061	.reset = mx31_reset,
1062	.fifo_size = 8,
1063	.has_dmamode = true,
1064	.dynamic_burst = false,
1065	.has_targetmode = false,
1066	.devtype = IMX35_CSPI,
1067	};
1068
1069	static struct spi_imx_devtype_data imx51_ecspi_devtype_data = {
1070	.intctrl = mx51_ecspi_intctrl,
1071	.prepare_message = mx51_ecspi_prepare_message,
1072	.prepare_transfer = mx51_ecspi_prepare_transfer,
1073	.trigger = mx51_ecspi_trigger,
1074	.rx_available = mx51_ecspi_rx_available,
1075	.reset = mx51_ecspi_reset,
1076	.setup_wml = mx51_setup_wml,
1077	.fifo_size = 64,
1078	.has_dmamode = true,
1079	.dynamic_burst = true,
1080	.has_targetmode = true,
1081	.disable = mx51_ecspi_disable,
1082	.devtype = IMX51_ECSPI,
1083	};
1084
1085	static struct spi_imx_devtype_data imx53_ecspi_devtype_data = {
1086	.intctrl = mx51_ecspi_intctrl,
1087	.prepare_message = mx51_ecspi_prepare_message,
1088	.prepare_transfer = mx51_ecspi_prepare_transfer,
1089	.trigger = mx51_ecspi_trigger,
1090	.rx_available = mx51_ecspi_rx_available,
1091	.reset = mx51_ecspi_reset,
1092	.fifo_size = 64,
1093	.has_dmamode = true,
1094	.has_targetmode = true,
1095	.disable = mx51_ecspi_disable,
1096	.devtype = IMX53_ECSPI,
1097	};
1098
1099	static struct spi_imx_devtype_data imx6ul_ecspi_devtype_data = {
1100	.intctrl = mx51_ecspi_intctrl,
1101	.prepare_message = mx51_ecspi_prepare_message,
1102	.prepare_transfer = mx51_ecspi_prepare_transfer,
1103	.trigger = mx51_ecspi_trigger,
1104	.rx_available = mx51_ecspi_rx_available,
1105	.reset = mx51_ecspi_reset,
1106	.setup_wml = mx51_setup_wml,
1107	.fifo_size = 64,
1108	.has_dmamode = true,
1109	.dynamic_burst = true,
1110	.has_targetmode = true,
1111	.tx_glitch_fixed = true,
1112	.disable = mx51_ecspi_disable,
1113	.devtype = IMX51_ECSPI,
1114	};
1115
1116	static const struct of_device_id spi_imx_dt_ids[] = {
1117	{ .compatible = "fsl,imx1-cspi", .data = &imx1_cspi_devtype_data, },
1118	{ .compatible = "fsl,imx21-cspi", .data = &imx21_cspi_devtype_data, },
1119	{ .compatible = "fsl,imx27-cspi", .data = &imx27_cspi_devtype_data, },
1120	{ .compatible = "fsl,imx31-cspi", .data = &imx31_cspi_devtype_data, },
1121	{ .compatible = "fsl,imx35-cspi", .data = &imx35_cspi_devtype_data, },
1122	{ .compatible = "fsl,imx51-ecspi", .data = &imx51_ecspi_devtype_data, },
1123	{ .compatible = "fsl,imx53-ecspi", .data = &imx53_ecspi_devtype_data, },
1124	{ .compatible = "fsl,imx6ul-ecspi", .data = &imx6ul_ecspi_devtype_data, },
1125	{ /* sentinel */ }
1126	};
1127	MODULE_DEVICE_TABLE(of, spi_imx_dt_ids);
1128
1129	static void spi_imx_set_burst_len(struct spi_imx_data *spi_imx, int n_bits)
1130	{
1131	u32 ctrl;
1132
1133	ctrl = readl(addr: spi_imx->base + MX51_ECSPI_CTRL);
1134	ctrl &= ~MX51_ECSPI_CTRL_BL_MASK;
1135	ctrl |= ((n_bits - 1) << MX51_ECSPI_CTRL_BL_OFFSET);
1136	writel(val: ctrl, addr: spi_imx->base + MX51_ECSPI_CTRL);
1137	}
1138
1139	static void spi_imx_push(struct spi_imx_data *spi_imx)
1140	{
1141	unsigned int burst_len;
1142
1143	/*
1144	* Reload the FIFO when the remaining bytes to be transferred in the
1145	* current burst is 0. This only applies when bits_per_word is a
1146	* multiple of 8.
1147	*/
1148	if (!spi_imx->remainder) {
1149	if (spi_imx->dynamic_burst) {
1150
1151	/* We need to deal unaligned data first */
1152	burst_len = spi_imx->count % MX51_ECSPI_CTRL_MAX_BURST;
1153
1154	if (!burst_len)
1155	burst_len = MX51_ECSPI_CTRL_MAX_BURST;
1156
1157	spi_imx_set_burst_len(spi_imx, n_bits: burst_len * 8);
1158
1159	spi_imx->remainder = burst_len;
1160	} else {
1161	spi_imx->remainder = spi_imx_bytes_per_word(bits_per_word: spi_imx->bits_per_word);
1162	}
1163	}
1164
1165	while (spi_imx->txfifo < spi_imx->devtype_data->fifo_size) {
1166	if (!spi_imx->count)
1167	break;
1168	if (spi_imx->dynamic_burst &&
1169	spi_imx->txfifo >= DIV_ROUND_UP(spi_imx->remainder, 4))
1170	break;
1171	spi_imx->tx(spi_imx);
1172	spi_imx->txfifo++;
1173	}
1174
1175	if (!spi_imx->target_mode)
1176	spi_imx->devtype_data->trigger(spi_imx);
1177	}
1178
1179	static irqreturn_t spi_imx_isr(int irq, void *dev_id)
1180	{
1181	struct spi_imx_data *spi_imx = dev_id;
1182
1183	while (spi_imx->txfifo &&
1184	spi_imx->devtype_data->rx_available(spi_imx)) {
1185	spi_imx->rx(spi_imx);
1186	spi_imx->txfifo--;
1187	}
1188
1189	if (spi_imx->count) {
1190	spi_imx_push(spi_imx);
1191	return IRQ_HANDLED;
1192	}
1193
1194	if (spi_imx->txfifo) {
1195	/* No data left to push, but still waiting for rx data,
1196	* enable receive data available interrupt.
1197	*/
1198	spi_imx->devtype_data->intctrl(
1199	spi_imx, MXC_INT_RR);
1200	return IRQ_HANDLED;
1201	}
1202
1203	spi_imx->devtype_data->intctrl(spi_imx, 0);
1204	complete(&spi_imx->xfer_done);
1205
1206	return IRQ_HANDLED;
1207	}
1208
1209	static int spi_imx_dma_configure(struct spi_controller *controller)
1210	{
1211	int ret;
1212	enum dma_slave_buswidth buswidth;
1213	struct dma_slave_config rx = {}, tx = {};
1214	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: controller);
1215
1216	switch (spi_imx_bytes_per_word(bits_per_word: spi_imx->bits_per_word)) {
1217	case 4:
1218	buswidth = DMA_SLAVE_BUSWIDTH_4_BYTES;
1219	break;
1220	case 2:
1221	buswidth = DMA_SLAVE_BUSWIDTH_2_BYTES;
1222	break;
1223	case 1:
1224	buswidth = DMA_SLAVE_BUSWIDTH_1_BYTE;
1225	break;
1226	default:
1227	return -EINVAL;
1228	}
1229
1230	tx.direction = DMA_MEM_TO_DEV;
1231	tx.dst_addr = spi_imx->base_phys + MXC_CSPITXDATA;
1232	tx.dst_addr_width = buswidth;
1233	tx.dst_maxburst = spi_imx->wml;
1234	ret = dmaengine_slave_config(chan: controller->dma_tx, config: &tx);
1235	if (ret) {
1236	dev_err(spi_imx->dev, "TX dma configuration failed with %d\n", ret);
1237	return ret;
1238	}
1239
1240	rx.direction = DMA_DEV_TO_MEM;
1241	rx.src_addr = spi_imx->base_phys + MXC_CSPIRXDATA;
1242	rx.src_addr_width = buswidth;
1243	rx.src_maxburst = spi_imx->wml;
1244	ret = dmaengine_slave_config(chan: controller->dma_rx, config: &rx);
1245	if (ret) {
1246	dev_err(spi_imx->dev, "RX dma configuration failed with %d\n", ret);
1247	return ret;
1248	}
1249
1250	return 0;
1251	}
1252
1253	static int spi_imx_setupxfer(struct spi_device *spi,
1254	struct spi_transfer *t)
1255	{
1256	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: spi->controller);
1257
1258	if (!t)
1259	return 0;
1260
1261	if (!t->speed_hz) {
1262	if (!spi->max_speed_hz) {
1263	dev_err(&spi->dev, "no speed_hz provided!\n");
1264	return -EINVAL;
1265	}
1266	dev_dbg(&spi->dev, "using spi->max_speed_hz!\n");
1267	spi_imx->spi_bus_clk = spi->max_speed_hz;
1268	} else
1269	spi_imx->spi_bus_clk = t->speed_hz;
1270
1271	spi_imx->bits_per_word = t->bits_per_word;
1272	spi_imx->count = t->len;
1273
1274	/*
1275	* Initialize the functions for transfer. To transfer non byte-aligned
1276	* words, we have to use multiple word-size bursts, we can't use
1277	* dynamic_burst in that case.
1278	*/
1279	if (spi_imx->devtype_data->dynamic_burst && !spi_imx->target_mode &&
1280	!(spi->mode & SPI_CS_WORD) &&
1281	(spi_imx->bits_per_word == 8 ||
1282	spi_imx->bits_per_word == 16 ||
1283	spi_imx->bits_per_word == 32)) {
1284
1285	spi_imx->rx = spi_imx_buf_rx_swap;
1286	spi_imx->tx = spi_imx_buf_tx_swap;
1287	spi_imx->dynamic_burst = 1;
1288
1289	} else {
1290	if (spi_imx->bits_per_word <= 8) {
1291	spi_imx->rx = spi_imx_buf_rx_u8;
1292	spi_imx->tx = spi_imx_buf_tx_u8;
1293	} else if (spi_imx->bits_per_word <= 16) {
1294	spi_imx->rx = spi_imx_buf_rx_u16;
1295	spi_imx->tx = spi_imx_buf_tx_u16;
1296	} else {
1297	spi_imx->rx = spi_imx_buf_rx_u32;
1298	spi_imx->tx = spi_imx_buf_tx_u32;
1299	}
1300	spi_imx->dynamic_burst = 0;
1301	}
1302
1303	if (spi_imx_can_dma(controller: spi_imx->controller, spi, transfer: t))
1304	spi_imx->usedma = true;
1305	else
1306	spi_imx->usedma = false;
1307
1308	spi_imx->rx_only = ((t->tx_buf == NULL)
1309	|| (t->tx_buf == spi->controller->dummy_tx));
1310
1311	if (is_imx53_ecspi(d: spi_imx) && spi_imx->target_mode) {
1312	spi_imx->rx = mx53_ecspi_rx_target;
1313	spi_imx->tx = mx53_ecspi_tx_target;
1314	spi_imx->target_burst = t->len;
1315	}
1316
1317	spi_imx->devtype_data->prepare_transfer(spi_imx, spi);
1318
1319	return 0;
1320	}
1321
1322	static void spi_imx_sdma_exit(struct spi_imx_data *spi_imx)
1323	{
1324	struct spi_controller *controller = spi_imx->controller;
1325
1326	if (controller->dma_rx) {
1327	dma_release_channel(chan: controller->dma_rx);
1328	controller->dma_rx = NULL;
1329	}
1330
1331	if (controller->dma_tx) {
1332	dma_release_channel(chan: controller->dma_tx);
1333	controller->dma_tx = NULL;
1334	}
1335	}
1336
1337	static int spi_imx_sdma_init(struct device *dev, struct spi_imx_data *spi_imx,
1338	struct spi_controller *controller)
1339	{
1340	int ret;
1341
1342	spi_imx->wml = spi_imx->devtype_data->fifo_size / 2;
1343
1344	/* Prepare for TX DMA: */
1345	controller->dma_tx = dma_request_chan(dev, name: "tx");
1346	if (IS_ERR(ptr: controller->dma_tx)) {
1347	ret = PTR_ERR(ptr: controller->dma_tx);
1348	dev_err_probe(dev, err: ret, fmt: "can't get the TX DMA channel!\n");
1349	controller->dma_tx = NULL;
1350	goto err;
1351	}
1352
1353	/* Prepare for RX : */
1354	controller->dma_rx = dma_request_chan(dev, name: "rx");
1355	if (IS_ERR(ptr: controller->dma_rx)) {
1356	ret = PTR_ERR(ptr: controller->dma_rx);
1357	dev_err_probe(dev, err: ret, fmt: "can't get the RX DMA channel!\n");
1358	controller->dma_rx = NULL;
1359	goto err;
1360	}
1361
1362	init_completion(x: &spi_imx->dma_rx_completion);
1363	init_completion(x: &spi_imx->dma_tx_completion);
1364	controller->can_dma = spi_imx_can_dma;
1365	controller->max_dma_len = MAX_SDMA_BD_BYTES;
1366	spi_imx->controller->flags = SPI_CONTROLLER_MUST_RX |
1367	SPI_CONTROLLER_MUST_TX;
1368
1369	return 0;
1370	err:
1371	spi_imx_sdma_exit(spi_imx);
1372	return ret;
1373	}
1374
1375	static void spi_imx_dma_rx_callback(void *cookie)
1376	{
1377	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1378
1379	complete(&spi_imx->dma_rx_completion);
1380	}
1381
1382	static void spi_imx_dma_tx_callback(void *cookie)
1383	{
1384	struct spi_imx_data *spi_imx = (struct spi_imx_data *)cookie;
1385
1386	complete(&spi_imx->dma_tx_completion);
1387	}
1388
1389	static int spi_imx_calculate_timeout(struct spi_imx_data *spi_imx, int size)
1390	{
1391	unsigned long timeout = 0;
1392
1393	/* Time with actual data transfer and CS change delay related to HW */
1394	timeout = (8 + 4) * size / spi_imx->spi_bus_clk;
1395
1396	/* Add extra second for scheduler related activities */
1397	timeout += 1;
1398
1399	/* Double calculated timeout */
1400	return msecs_to_jiffies(m: 2 * timeout * MSEC_PER_SEC);
1401	}
1402
1403	static int spi_imx_dma_transfer(struct spi_imx_data *spi_imx,
1404	struct spi_transfer *transfer)
1405	{
1406	struct dma_async_tx_descriptor *desc_tx, *desc_rx;
1407	unsigned long transfer_timeout;
1408	unsigned long timeout;
1409	struct spi_controller *controller = spi_imx->controller;
1410	struct sg_table *tx = &transfer->tx_sg, *rx = &transfer->rx_sg;
1411	struct scatterlist *last_sg = sg_last(s: rx->sgl, rx->nents);
1412	unsigned int bytes_per_word, i;
1413	int ret;
1414
1415	/* Get the right burst length from the last sg to ensure no tail data */
1416	bytes_per_word = spi_imx_bytes_per_word(bits_per_word: transfer->bits_per_word);
1417	for (i = spi_imx->devtype_data->fifo_size / 2; i > 0; i--) {
1418	if (!(sg_dma_len(last_sg) % (i * bytes_per_word)))
1419	break;
1420	}
1421	/* Use 1 as wml in case no available burst length got */
1422	if (i == 0)
1423	i = 1;
1424
1425	spi_imx->wml = i;
1426
1427	ret = spi_imx_dma_configure(controller);
1428	if (ret)
1429	goto dma_failure_no_start;
1430
1431	if (!spi_imx->devtype_data->setup_wml) {
1432	dev_err(spi_imx->dev, "No setup_wml()?\n");
1433	ret = -EINVAL;
1434	goto dma_failure_no_start;
1435	}
1436	spi_imx->devtype_data->setup_wml(spi_imx);
1437
1438	/*
1439	* The TX DMA setup starts the transfer, so make sure RX is configured
1440	* before TX.
1441	*/
1442	desc_rx = dmaengine_prep_slave_sg(chan: controller->dma_rx,
1443	sgl: rx->sgl, sg_len: rx->nents, dir: DMA_DEV_TO_MEM,
1444	flags: DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1445	if (!desc_rx) {
1446	ret = -EINVAL;
1447	goto dma_failure_no_start;
1448	}
1449
1450	desc_rx->callback = spi_imx_dma_rx_callback;
1451	desc_rx->callback_param = (void *)spi_imx;
1452	dmaengine_submit(desc: desc_rx);
1453	reinit_completion(x: &spi_imx->dma_rx_completion);
1454	dma_async_issue_pending(chan: controller->dma_rx);
1455
1456	desc_tx = dmaengine_prep_slave_sg(chan: controller->dma_tx,
1457	sgl: tx->sgl, sg_len: tx->nents, dir: DMA_MEM_TO_DEV,
1458	flags: DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1459	if (!desc_tx) {
1460	dmaengine_terminate_all(chan: controller->dma_tx);
1461	dmaengine_terminate_all(chan: controller->dma_rx);
1462	return -EINVAL;
1463	}
1464
1465	desc_tx->callback = spi_imx_dma_tx_callback;
1466	desc_tx->callback_param = (void *)spi_imx;
1467	dmaengine_submit(desc: desc_tx);
1468	reinit_completion(x: &spi_imx->dma_tx_completion);
1469	dma_async_issue_pending(chan: controller->dma_tx);
1470
1471	transfer_timeout = spi_imx_calculate_timeout(spi_imx, size: transfer->len);
1472
1473	/* Wait SDMA to finish the data transfer.*/
1474	timeout = wait_for_completion_timeout(x: &spi_imx->dma_tx_completion,
1475	timeout: transfer_timeout);
1476	if (!timeout) {
1477	dev_err(spi_imx->dev, "I/O Error in DMA TX\n");
1478	dmaengine_terminate_all(chan: controller->dma_tx);
1479	dmaengine_terminate_all(chan: controller->dma_rx);
1480	return -ETIMEDOUT;
1481	}
1482
1483	timeout = wait_for_completion_timeout(x: &spi_imx->dma_rx_completion,
1484	timeout: transfer_timeout);
1485	if (!timeout) {
1486	dev_err(&controller->dev, "I/O Error in DMA RX\n");
1487	spi_imx->devtype_data->reset(spi_imx);
1488	dmaengine_terminate_all(chan: controller->dma_rx);
1489	return -ETIMEDOUT;
1490	}
1491
1492	return 0;
1493	/* fallback to pio */
1494	dma_failure_no_start:
1495	transfer->error |= SPI_TRANS_FAIL_NO_START;
1496	return ret;
1497	}
1498
1499	static int spi_imx_pio_transfer(struct spi_device *spi,
1500	struct spi_transfer *transfer)
1501	{
1502	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: spi->controller);
1503	unsigned long transfer_timeout;
1504	unsigned long timeout;
1505
1506	spi_imx->tx_buf = transfer->tx_buf;
1507	spi_imx->rx_buf = transfer->rx_buf;
1508	spi_imx->count = transfer->len;
1509	spi_imx->txfifo = 0;
1510	spi_imx->remainder = 0;
1511
1512	reinit_completion(x: &spi_imx->xfer_done);
1513
1514	spi_imx_push(spi_imx);
1515
1516	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE);
1517
1518	transfer_timeout = spi_imx_calculate_timeout(spi_imx, size: transfer->len);
1519
1520	timeout = wait_for_completion_timeout(x: &spi_imx->xfer_done,
1521	timeout: transfer_timeout);
1522	if (!timeout) {
1523	dev_err(&spi->dev, "I/O Error in PIO\n");
1524	spi_imx->devtype_data->reset(spi_imx);
1525	return -ETIMEDOUT;
1526	}
1527
1528	return 0;
1529	}
1530
1531	static int spi_imx_poll_transfer(struct spi_device *spi,
1532	struct spi_transfer *transfer)
1533	{
1534	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: spi->controller);
1535	unsigned long timeout;
1536
1537	spi_imx->tx_buf = transfer->tx_buf;
1538	spi_imx->rx_buf = transfer->rx_buf;
1539	spi_imx->count = transfer->len;
1540	spi_imx->txfifo = 0;
1541	spi_imx->remainder = 0;
1542
1543	/* fill in the fifo before timeout calculations if we are
1544	* interrupted here, then the data is getting transferred by
1545	* the HW while we are interrupted
1546	*/
1547	spi_imx_push(spi_imx);
1548
1549	timeout = spi_imx_calculate_timeout(spi_imx, size: transfer->len) + jiffies;
1550	while (spi_imx->txfifo) {
1551	/* RX */
1552	while (spi_imx->txfifo &&
1553	spi_imx->devtype_data->rx_available(spi_imx)) {
1554	spi_imx->rx(spi_imx);
1555	spi_imx->txfifo--;
1556	}
1557
1558	/* TX */
1559	if (spi_imx->count) {
1560	spi_imx_push(spi_imx);
1561	continue;
1562	}
1563
1564	if (spi_imx->txfifo &&
1565	time_after(jiffies, timeout)) {
1566
1567	dev_err_ratelimited(&spi->dev,
1568	"timeout period reached: jiffies: %lu- falling back to interrupt mode\n",
1569	jiffies - timeout);
1570
1571	/* fall back to interrupt mode */
1572	return spi_imx_pio_transfer(spi, transfer);
1573	}
1574	}
1575
1576	return 0;
1577	}
1578
1579	static int spi_imx_pio_transfer_target(struct spi_device *spi,
1580	struct spi_transfer *transfer)
1581	{
1582	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: spi->controller);
1583	int ret = 0;
1584
1585	if (is_imx53_ecspi(d: spi_imx) &&
1586	transfer->len > MX53_MAX_TRANSFER_BYTES) {
1587	dev_err(&spi->dev, "Transaction too big, max size is %d bytes\n",
1588	MX53_MAX_TRANSFER_BYTES);
1589	return -EMSGSIZE;
1590	}
1591
1592	spi_imx->tx_buf = transfer->tx_buf;
1593	spi_imx->rx_buf = transfer->rx_buf;
1594	spi_imx->count = transfer->len;
1595	spi_imx->txfifo = 0;
1596	spi_imx->remainder = 0;
1597
1598	reinit_completion(x: &spi_imx->xfer_done);
1599	spi_imx->target_aborted = false;
1600
1601	spi_imx_push(spi_imx);
1602
1603	spi_imx->devtype_data->intctrl(spi_imx, MXC_INT_TE | MXC_INT_RDR);
1604
1605	if (wait_for_completion_interruptible(x: &spi_imx->xfer_done) ||
1606	spi_imx->target_aborted) {
1607	dev_dbg(&spi->dev, "interrupted\n");
1608	ret = -EINTR;
1609	}
1610
1611	/* ecspi has a HW issue when works in Target mode,
1612	* after 64 words writtern to TXFIFO, even TXFIFO becomes empty,
1613	* ECSPI_TXDATA keeps shift out the last word data,
1614	* so we have to disable ECSPI when in target mode after the
1615	* transfer completes
1616	*/
1617	if (spi_imx->devtype_data->disable)
1618	spi_imx->devtype_data->disable(spi_imx);
1619
1620	return ret;
1621	}
1622
1623	static int spi_imx_transfer_one(struct spi_controller *controller,
1624	struct spi_device *spi,
1625	struct spi_transfer *transfer)
1626	{
1627	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: spi->controller);
1628	unsigned long hz_per_byte, byte_limit;
1629
1630	spi_imx_setupxfer(spi, t: transfer);
1631	transfer->effective_speed_hz = spi_imx->spi_bus_clk;
1632
1633	/* flush rxfifo before transfer */
1634	while (spi_imx->devtype_data->rx_available(spi_imx))
1635	readl(addr: spi_imx->base + MXC_CSPIRXDATA);
1636
1637	if (spi_imx->target_mode)
1638	return spi_imx_pio_transfer_target(spi, transfer);
1639
1640	/*
1641	* If we decided in spi_imx_can_dma() that we want to do a DMA
1642	* transfer, the SPI transfer has already been mapped, so we
1643	* have to do the DMA transfer here.
1644	*/
1645	if (spi_imx->usedma)
1646	return spi_imx_dma_transfer(spi_imx, transfer);
1647	/*
1648	* Calculate the estimated time in us the transfer runs. Find
1649	* the number of Hz per byte per polling limit.
1650	*/
1651	hz_per_byte = polling_limit_us ? ((8 + 4) * USEC_PER_SEC) / polling_limit_us : 0;
1652	byte_limit = hz_per_byte ? transfer->effective_speed_hz / hz_per_byte : 1;
1653
1654	/* run in polling mode for short transfers */
1655	if (transfer->len < byte_limit)
1656	return spi_imx_poll_transfer(spi, transfer);
1657
1658	return spi_imx_pio_transfer(spi, transfer);
1659	}
1660
1661	static int spi_imx_setup(struct spi_device *spi)
1662	{
1663	dev_dbg(&spi->dev, "%s: mode %d, %u bpw, %d hz\n", __func__,
1664	spi->mode, spi->bits_per_word, spi->max_speed_hz);
1665
1666	return 0;
1667	}
1668
1669	static void spi_imx_cleanup(struct spi_device *spi)
1670	{
1671	}
1672
1673	static int
1674	spi_imx_prepare_message(struct spi_controller *controller, struct spi_message *msg)
1675	{
1676	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: controller);
1677	int ret;
1678
1679	ret = pm_runtime_resume_and_get(dev: spi_imx->dev);
1680	if (ret < 0) {
1681	dev_err(spi_imx->dev, "failed to enable clock\n");
1682	return ret;
1683	}
1684
1685	ret = spi_imx->devtype_data->prepare_message(spi_imx, msg);
1686	if (ret) {
1687	pm_runtime_mark_last_busy(dev: spi_imx->dev);
1688	pm_runtime_put_autosuspend(dev: spi_imx->dev);
1689	}
1690
1691	return ret;
1692	}
1693
1694	static int
1695	spi_imx_unprepare_message(struct spi_controller *controller, struct spi_message *msg)
1696	{
1697	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: controller);
1698
1699	pm_runtime_mark_last_busy(dev: spi_imx->dev);
1700	pm_runtime_put_autosuspend(dev: spi_imx->dev);
1701	return 0;
1702	}
1703
1704	static int spi_imx_target_abort(struct spi_controller *controller)
1705	{
1706	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: controller);
1707
1708	spi_imx->target_aborted = true;
1709	complete(&spi_imx->xfer_done);
1710
1711	return 0;
1712	}
1713
1714	static int spi_imx_probe(struct platform_device *pdev)
1715	{
1716	struct device_node *np = pdev->dev.of_node;
1717	struct spi_controller *controller;
1718	struct spi_imx_data *spi_imx;
1719	struct resource *res;
1720	int ret, irq, spi_drctl;
1721	const struct spi_imx_devtype_data *devtype_data =
1722	of_device_get_match_data(dev: &pdev->dev);
1723	bool target_mode;
1724	u32 val;
1725
1726	target_mode = devtype_data->has_targetmode &&
1727	of_property_read_bool(np, propname: "spi-slave");
1728	if (target_mode)
1729	controller = spi_alloc_target(dev: &pdev->dev,
1730	size: sizeof(struct spi_imx_data));
1731	else
1732	controller = spi_alloc_host(dev: &pdev->dev,
1733	size: sizeof(struct spi_imx_data));
1734	if (!controller)
1735	return -ENOMEM;
1736
1737	ret = of_property_read_u32(np, propname: "fsl,spi-rdy-drctl", out_value: &spi_drctl);
1738	if ((ret < 0) || (spi_drctl >= 0x3)) {
1739	/* '11' is reserved */
1740	spi_drctl = 0;
1741	}
1742
1743	platform_set_drvdata(pdev, data: controller);
1744
1745	controller->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32);
1746	controller->bus_num = np ? -1 : pdev->id;
1747	controller->use_gpio_descriptors = true;
1748
1749	spi_imx = spi_controller_get_devdata(ctlr: controller);
1750	spi_imx->controller = controller;
1751	spi_imx->dev = &pdev->dev;
1752	spi_imx->target_mode = target_mode;
1753
1754	spi_imx->devtype_data = devtype_data;
1755
1756	/*
1757	* Get number of chip selects from device properties. This can be
1758	* coming from device tree or boardfiles, if it is not defined,
1759	* a default value of 3 chip selects will be used, as all the legacy
1760	* board files have <= 3 chip selects.
1761	*/
1762	if (!device_property_read_u32(dev: &pdev->dev, propname: "num-cs", val: &val))
1763	controller->num_chipselect = val;
1764	else
1765	controller->num_chipselect = 3;
1766
1767	controller->transfer_one = spi_imx_transfer_one;
1768	controller->setup = spi_imx_setup;
1769	controller->cleanup = spi_imx_cleanup;
1770	controller->prepare_message = spi_imx_prepare_message;
1771	controller->unprepare_message = spi_imx_unprepare_message;
1772	controller->target_abort = spi_imx_target_abort;
1773	controller->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_NO_CS |
1774	SPI_MOSI_IDLE_LOW;
1775
1776	if (is_imx35_cspi(d: spi_imx) || is_imx51_ecspi(d: spi_imx) ||
1777	is_imx53_ecspi(d: spi_imx))
1778	controller->mode_bits |= SPI_LOOP | SPI_READY;
1779
1780	if (is_imx51_ecspi(d: spi_imx) || is_imx53_ecspi(d: spi_imx))
1781	controller->mode_bits |= SPI_RX_CPHA_FLIP;
1782
1783	if (is_imx51_ecspi(d: spi_imx) &&
1784	device_property_read_u32(dev: &pdev->dev, propname: "cs-gpios", NULL))
1785	/*
1786	* When using HW-CS implementing SPI_CS_WORD can be done by just
1787	* setting the burst length to the word size. This is
1788	* considerably faster than manually controlling the CS.
1789	*/
1790	controller->mode_bits |= SPI_CS_WORD;
1791
1792	if (is_imx51_ecspi(d: spi_imx) || is_imx53_ecspi(d: spi_imx)) {
1793	controller->max_native_cs = 4;
1794	controller->flags |= SPI_CONTROLLER_GPIO_SS;
1795	}
1796
1797	spi_imx->spi_drctl = spi_drctl;
1798
1799	init_completion(x: &spi_imx->xfer_done);
1800
1801	spi_imx->base = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
1802	if (IS_ERR(ptr: spi_imx->base)) {
1803	ret = PTR_ERR(ptr: spi_imx->base);
1804	goto out_controller_put;
1805	}
1806	spi_imx->base_phys = res->start;
1807
1808	irq = platform_get_irq(pdev, 0);
1809	if (irq < 0) {
1810	ret = irq;
1811	goto out_controller_put;
1812	}
1813
1814	ret = devm_request_irq(dev: &pdev->dev, irq, handler: spi_imx_isr, irqflags: 0,
1815	devname: dev_name(dev: &pdev->dev), dev_id: spi_imx);
1816	if (ret) {
1817	dev_err(&pdev->dev, "can't get irq%d: %d\n", irq, ret);
1818	goto out_controller_put;
1819	}
1820
1821	spi_imx->clk_ipg = devm_clk_get(dev: &pdev->dev, id: "ipg");
1822	if (IS_ERR(ptr: spi_imx->clk_ipg)) {
1823	ret = PTR_ERR(ptr: spi_imx->clk_ipg);
1824	goto out_controller_put;
1825	}
1826
1827	spi_imx->clk_per = devm_clk_get(dev: &pdev->dev, id: "per");
1828	if (IS_ERR(ptr: spi_imx->clk_per)) {
1829	ret = PTR_ERR(ptr: spi_imx->clk_per);
1830	goto out_controller_put;
1831	}
1832
1833	ret = clk_prepare_enable(clk: spi_imx->clk_per);
1834	if (ret)
1835	goto out_controller_put;
1836
1837	ret = clk_prepare_enable(clk: spi_imx->clk_ipg);
1838	if (ret)
1839	goto out_put_per;
1840
1841	pm_runtime_set_autosuspend_delay(dev: spi_imx->dev, MXC_RPM_TIMEOUT);
1842	pm_runtime_use_autosuspend(dev: spi_imx->dev);
1843	pm_runtime_get_noresume(dev: spi_imx->dev);
1844	pm_runtime_set_active(dev: spi_imx->dev);
1845	pm_runtime_enable(dev: spi_imx->dev);
1846
1847	spi_imx->spi_clk = clk_get_rate(clk: spi_imx->clk_per);
1848	/*
1849	* Only validated on i.mx35 and i.mx6 now, can remove the constraint
1850	* if validated on other chips.
1851	*/
1852	if (spi_imx->devtype_data->has_dmamode) {
1853	ret = spi_imx_sdma_init(dev: &pdev->dev, spi_imx, controller);
1854	if (ret == -EPROBE_DEFER)
1855	goto out_runtime_pm_put;
1856
1857	if (ret < 0)
1858	dev_dbg(&pdev->dev, "dma setup error %d, use pio\n",
1859	ret);
1860	}
1861
1862	spi_imx->devtype_data->reset(spi_imx);
1863
1864	spi_imx->devtype_data->intctrl(spi_imx, 0);
1865
1866	controller->dev.of_node = pdev->dev.of_node;
1867	ret = spi_register_controller(ctlr: controller);
1868	if (ret) {
1869	dev_err_probe(dev: &pdev->dev, err: ret, fmt: "register controller failed\n");
1870	goto out_register_controller;
1871	}
1872
1873	pm_runtime_mark_last_busy(dev: spi_imx->dev);
1874	pm_runtime_put_autosuspend(dev: spi_imx->dev);
1875
1876	return ret;
1877
1878	out_register_controller:
1879	if (spi_imx->devtype_data->has_dmamode)
1880	spi_imx_sdma_exit(spi_imx);
1881	out_runtime_pm_put:
1882	pm_runtime_dont_use_autosuspend(dev: spi_imx->dev);
1883	pm_runtime_set_suspended(dev: &pdev->dev);
1884	pm_runtime_disable(dev: spi_imx->dev);
1885
1886	clk_disable_unprepare(clk: spi_imx->clk_ipg);
1887	out_put_per:
1888	clk_disable_unprepare(clk: spi_imx->clk_per);
1889	out_controller_put:
1890	spi_controller_put(ctlr: controller);
1891
1892	return ret;
1893	}
1894
1895	static void spi_imx_remove(struct platform_device *pdev)
1896	{
1897	struct spi_controller *controller = platform_get_drvdata(pdev);
1898	struct spi_imx_data *spi_imx = spi_controller_get_devdata(ctlr: controller);
1899	int ret;
1900
1901	spi_unregister_controller(ctlr: controller);
1902
1903	ret = pm_runtime_get_sync(dev: spi_imx->dev);
1904	if (ret >= 0)
1905	writel(val: 0, addr: spi_imx->base + MXC_CSPICTRL);
1906	else
1907	dev_warn(spi_imx->dev, "failed to enable clock, skip hw disable\n");
1908
1909	pm_runtime_dont_use_autosuspend(dev: spi_imx->dev);
1910	pm_runtime_put_sync(dev: spi_imx->dev);
1911	pm_runtime_disable(dev: spi_imx->dev);
1912
1913	spi_imx_sdma_exit(spi_imx);
1914	}
1915
1916	static int __maybe_unused spi_imx_runtime_resume(struct device *dev)
1917	{
1918	struct spi_controller *controller = dev_get_drvdata(dev);
1919	struct spi_imx_data *spi_imx;
1920	int ret;
1921
1922	spi_imx = spi_controller_get_devdata(ctlr: controller);
1923
1924	ret = clk_prepare_enable(clk: spi_imx->clk_per);
1925	if (ret)
1926	return ret;
1927
1928	ret = clk_prepare_enable(clk: spi_imx->clk_ipg);
1929	if (ret) {
1930	clk_disable_unprepare(clk: spi_imx->clk_per);
1931	return ret;
1932	}
1933
1934	return 0;
1935	}
1936
1937	static int __maybe_unused spi_imx_runtime_suspend(struct device *dev)
1938	{
1939	struct spi_controller *controller = dev_get_drvdata(dev);
1940	struct spi_imx_data *spi_imx;
1941
1942	spi_imx = spi_controller_get_devdata(ctlr: controller);
1943
1944	clk_disable_unprepare(clk: spi_imx->clk_per);
1945	clk_disable_unprepare(clk: spi_imx->clk_ipg);
1946
1947	return 0;
1948	}
1949
1950	static int __maybe_unused spi_imx_suspend(struct device *dev)
1951	{
1952	pinctrl_pm_select_sleep_state(dev);
1953	return 0;
1954	}
1955
1956	static int __maybe_unused spi_imx_resume(struct device *dev)
1957	{
1958	pinctrl_pm_select_default_state(dev);
1959	return 0;
1960	}
1961
1962	static const struct dev_pm_ops imx_spi_pm = {
1963	SET_RUNTIME_PM_OPS(spi_imx_runtime_suspend,
1964	spi_imx_runtime_resume, NULL)
1965	SET_SYSTEM_SLEEP_PM_OPS(spi_imx_suspend, spi_imx_resume)
1966	};
1967
1968	static struct platform_driver spi_imx_driver = {
1969	.driver = {
1970	.name = DRIVER_NAME,
1971	.of_match_table = spi_imx_dt_ids,
1972	.pm = &imx_spi_pm,
1973	},
1974	.probe = spi_imx_probe,
1975	.remove_new = spi_imx_remove,
1976	};
1977	module_platform_driver(spi_imx_driver);
1978
1979	MODULE_DESCRIPTION("i.MX SPI Controller driver");
1980	MODULE_AUTHOR("Sascha Hauer, Pengutronix");
1981	MODULE_LICENSE("GPL");
1982	MODULE_ALIAS("platform:" DRIVER_NAME);
1983