1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* OMAP2 McSPI controller driver
4	*
5	* Copyright (C) 2005, 2006 Nokia Corporation
6	* Author: Samuel Ortiz <samuel.ortiz@nokia.com> and
7	* Juha Yrjola <juha.yrjola@nokia.com>
8	*/
9
10	#include <linux/kernel.h>
11	#include <linux/interrupt.h>
12	#include <linux/module.h>
13	#include <linux/device.h>
14	#include <linux/delay.h>
15	#include <linux/dma-mapping.h>
16	#include <linux/dmaengine.h>
17	#include <linux/pinctrl/consumer.h>
18	#include <linux/platform_device.h>
19	#include <linux/err.h>
20	#include <linux/clk.h>
21	#include <linux/io.h>
22	#include <linux/slab.h>
23	#include <linux/pm_runtime.h>
24	#include <linux/of.h>
25	#include <linux/of_device.h>
26	#include <linux/gcd.h>
27
28	#include <linux/spi/spi.h>
29
30	#include <linux/platform_data/spi-omap2-mcspi.h>
31
32	#define OMAP2_MCSPI_MAX_FREQ 48000000
33	#define OMAP2_MCSPI_MAX_DIVIDER 4096
34	#define OMAP2_MCSPI_MAX_FIFODEPTH 64
35	#define OMAP2_MCSPI_MAX_FIFOWCNT 0xFFFF
36	#define SPI_AUTOSUSPEND_TIMEOUT 2000
37
38	#define OMAP2_MCSPI_REVISION 0x00
39	#define OMAP2_MCSPI_SYSSTATUS 0x14
40	#define OMAP2_MCSPI_IRQSTATUS 0x18
41	#define OMAP2_MCSPI_IRQENABLE 0x1c
42	#define OMAP2_MCSPI_WAKEUPENABLE 0x20
43	#define OMAP2_MCSPI_SYST 0x24
44	#define OMAP2_MCSPI_MODULCTRL 0x28
45	#define OMAP2_MCSPI_XFERLEVEL 0x7c
46
47	/* per-channel banks, 0x14 bytes each, first is: */
48	#define OMAP2_MCSPI_CHCONF0 0x2c
49	#define OMAP2_MCSPI_CHSTAT0 0x30
50	#define OMAP2_MCSPI_CHCTRL0 0x34
51	#define OMAP2_MCSPI_TX0 0x38
52	#define OMAP2_MCSPI_RX0 0x3c
53
54	/* per-register bitmasks: */
55	#define OMAP2_MCSPI_IRQSTATUS_EOW BIT(17)
56
57	#define OMAP2_MCSPI_MODULCTRL_SINGLE BIT(0)
58	#define OMAP2_MCSPI_MODULCTRL_MS BIT(2)
59	#define OMAP2_MCSPI_MODULCTRL_STEST BIT(3)
60
61	#define OMAP2_MCSPI_CHCONF_PHA BIT(0)
62	#define OMAP2_MCSPI_CHCONF_POL BIT(1)
63	#define OMAP2_MCSPI_CHCONF_CLKD_MASK (0x0f << 2)
64	#define OMAP2_MCSPI_CHCONF_EPOL BIT(6)
65	#define OMAP2_MCSPI_CHCONF_WL_MASK (0x1f << 7)
66	#define OMAP2_MCSPI_CHCONF_TRM_RX_ONLY BIT(12)
67	#define OMAP2_MCSPI_CHCONF_TRM_TX_ONLY BIT(13)
68	#define OMAP2_MCSPI_CHCONF_TRM_MASK (0x03 << 12)
69	#define OMAP2_MCSPI_CHCONF_DMAW BIT(14)
70	#define OMAP2_MCSPI_CHCONF_DMAR BIT(15)
71	#define OMAP2_MCSPI_CHCONF_DPE0 BIT(16)
72	#define OMAP2_MCSPI_CHCONF_DPE1 BIT(17)
73	#define OMAP2_MCSPI_CHCONF_IS BIT(18)
74	#define OMAP2_MCSPI_CHCONF_TURBO BIT(19)
75	#define OMAP2_MCSPI_CHCONF_FORCE BIT(20)
76	#define OMAP2_MCSPI_CHCONF_FFET BIT(27)
77	#define OMAP2_MCSPI_CHCONF_FFER BIT(28)
78	#define OMAP2_MCSPI_CHCONF_CLKG BIT(29)
79
80	#define OMAP2_MCSPI_CHSTAT_RXS BIT(0)
81	#define OMAP2_MCSPI_CHSTAT_TXS BIT(1)
82	#define OMAP2_MCSPI_CHSTAT_EOT BIT(2)
83	#define OMAP2_MCSPI_CHSTAT_TXFFE BIT(3)
84
85	#define OMAP2_MCSPI_CHCTRL_EN BIT(0)
86	#define OMAP2_MCSPI_CHCTRL_EXTCLK_MASK (0xff << 8)
87
88	#define OMAP2_MCSPI_WAKEUPENABLE_WKEN BIT(0)
89
90	/* We have 2 DMA channels per CS, one for RX and one for TX */
91	struct omap2_mcspi_dma {
92	struct dma_chan *dma_tx;
93	struct dma_chan *dma_rx;
94
95	struct completion dma_tx_completion;
96	struct completion dma_rx_completion;
97
98	char dma_rx_ch_name[14];
99	char dma_tx_ch_name[14];
100	};
101
102	/* use PIO for small transfers, avoiding DMA setup/teardown overhead and
103	* cache operations; better heuristics consider wordsize and bitrate.
104	*/
105	#define DMA_MIN_BYTES 160
106
107
108	/*
109	* Used for context save and restore, structure members to be updated whenever
110	* corresponding registers are modified.
111	*/
112	struct omap2_mcspi_regs {
113	u32 modulctrl;
114	u32 wakeupenable;
115	struct list_head cs;
116	};
117
118	struct omap2_mcspi {
119	struct completion txdone;
120	struct spi_controller *ctlr;
121	/* Virtual base address of the controller */
122	void __iomem *base;
123	unsigned long phys;
124	/* SPI1 has 4 channels, while SPI2 has 2 */
125	struct omap2_mcspi_dma *dma_channels;
126	struct device *dev;
127	struct omap2_mcspi_regs ctx;
128	struct clk *ref_clk;
129	int fifo_depth;
130	bool target_aborted;
131	unsigned int pin_dir:1;
132	size_t max_xfer_len;
133	u32 ref_clk_hz;
134	};
135
136	struct omap2_mcspi_cs {
137	void __iomem *base;
138	unsigned long phys;
139	int word_len;
140	u16 mode;
141	struct list_head node;
142	/* Context save and restore shadow register */
143	u32 chconf0, chctrl0;
144	};
145
146	static inline void mcspi_write_reg(struct spi_controller *ctlr,
147	int idx, u32 val)
148	{
149	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
150
151	writel_relaxed(val, mcspi->base + idx);
152	}
153
154	static inline u32 mcspi_read_reg(struct spi_controller *ctlr, int idx)
155	{
156	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
157
158	return readl_relaxed(mcspi->base + idx);
159	}
160
161	static inline void mcspi_write_cs_reg(const struct spi_device *spi,
162	int idx, u32 val)
163	{
164	struct omap2_mcspi_cs *cs = spi->controller_state;
165
166	writel_relaxed(val, cs->base + idx);
167	}
168
169	static inline u32 mcspi_read_cs_reg(const struct spi_device *spi, int idx)
170	{
171	struct omap2_mcspi_cs *cs = spi->controller_state;
172
173	return readl_relaxed(cs->base + idx);
174	}
175
176	static inline u32 mcspi_cached_chconf0(const struct spi_device *spi)
177	{
178	struct omap2_mcspi_cs *cs = spi->controller_state;
179
180	return cs->chconf0;
181	}
182
183	static inline void mcspi_write_chconf0(const struct spi_device *spi, u32 val)
184	{
185	struct omap2_mcspi_cs *cs = spi->controller_state;
186
187	cs->chconf0 = val;
188	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCONF0, val);
189	mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCONF0);
190	}
191
192	static inline int mcspi_bytes_per_word(int word_len)
193	{
194	if (word_len <= 8)
195	return 1;
196	else if (word_len <= 16)
197	return 2;
198	else /* word_len <= 32 */
199	return 4;
200	}
201
202	static void omap2_mcspi_set_dma_req(const struct spi_device *spi,
203	int is_read, int enable)
204	{
205	u32 l, rw;
206
207	l = mcspi_cached_chconf0(spi);
208
209	if (is_read) /* 1 is read, 0 write */
210	rw = OMAP2_MCSPI_CHCONF_DMAR;
211	else
212	rw = OMAP2_MCSPI_CHCONF_DMAW;
213
214	if (enable)
215	l |= rw;
216	else
217	l &= ~rw;
218
219	mcspi_write_chconf0(spi, val: l);
220	}
221
222	static void omap2_mcspi_set_enable(const struct spi_device *spi, int enable)
223	{
224	struct omap2_mcspi_cs *cs = spi->controller_state;
225	u32 l;
226
227	l = cs->chctrl0;
228	if (enable)
229	l |= OMAP2_MCSPI_CHCTRL_EN;
230	else
231	l &= ~OMAP2_MCSPI_CHCTRL_EN;
232	cs->chctrl0 = l;
233	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, val: cs->chctrl0);
234	/* Flash post-writes */
235	mcspi_read_cs_reg(spi, OMAP2_MCSPI_CHCTRL0);
236	}
237
238	static void omap2_mcspi_set_cs(struct spi_device *spi, bool enable)
239	{
240	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr: spi->controller);
241	u32 l;
242
243	/* The controller handles the inverted chip selects
244	* using the OMAP2_MCSPI_CHCONF_EPOL bit so revert
245	* the inversion from the core spi_set_cs function.
246	*/
247	if (spi->mode & SPI_CS_HIGH)
248	enable = !enable;
249
250	if (spi->controller_state) {
251	int err = pm_runtime_resume_and_get(dev: mcspi->dev);
252	if (err < 0) {
253	dev_err(mcspi->dev, "failed to get sync: %d\n", err);
254	return;
255	}
256
257	l = mcspi_cached_chconf0(spi);
258
259	if (enable)
260	l &= ~OMAP2_MCSPI_CHCONF_FORCE;
261	else
262	l |= OMAP2_MCSPI_CHCONF_FORCE;
263
264	mcspi_write_chconf0(spi, val: l);
265
266	pm_runtime_mark_last_busy(dev: mcspi->dev);
267	pm_runtime_put_autosuspend(dev: mcspi->dev);
268	}
269	}
270
271	static void omap2_mcspi_set_mode(struct spi_controller *ctlr)
272	{
273	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
274	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
275	u32 l;
276
277	/*
278	* Choose host or target mode
279	*/
280	l = mcspi_read_reg(ctlr, OMAP2_MCSPI_MODULCTRL);
281	l &= ~(OMAP2_MCSPI_MODULCTRL_STEST);
282	if (spi_controller_is_target(ctlr)) {
283	l |= (OMAP2_MCSPI_MODULCTRL_MS);
284	} else {
285	l &= ~(OMAP2_MCSPI_MODULCTRL_MS);
286	l |= OMAP2_MCSPI_MODULCTRL_SINGLE;
287	}
288	mcspi_write_reg(ctlr, OMAP2_MCSPI_MODULCTRL, val: l);
289
290	ctx->modulctrl = l;
291	}
292
293	static void omap2_mcspi_set_fifo(const struct spi_device *spi,
294	struct spi_transfer *t, int enable)
295	{
296	struct spi_controller *ctlr = spi->controller;
297	struct omap2_mcspi_cs *cs = spi->controller_state;
298	struct omap2_mcspi *mcspi;
299	unsigned int wcnt;
300	int max_fifo_depth, bytes_per_word;
301	u32 chconf, xferlevel;
302
303	mcspi = spi_controller_get_devdata(ctlr);
304
305	chconf = mcspi_cached_chconf0(spi);
306	if (enable) {
307	bytes_per_word = mcspi_bytes_per_word(word_len: cs->word_len);
308	if (t->len % bytes_per_word != 0)
309	goto disable_fifo;
310
311	if (t->rx_buf != NULL && t->tx_buf != NULL)
312	max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH / 2;
313	else
314	max_fifo_depth = OMAP2_MCSPI_MAX_FIFODEPTH;
315
316	wcnt = t->len / bytes_per_word;
317	if (wcnt > OMAP2_MCSPI_MAX_FIFOWCNT)
318	goto disable_fifo;
319
320	xferlevel = wcnt << 16;
321	if (t->rx_buf != NULL) {
322	chconf |= OMAP2_MCSPI_CHCONF_FFER;
323	xferlevel |= (bytes_per_word - 1) << 8;
324	}
325
326	if (t->tx_buf != NULL) {
327	chconf |= OMAP2_MCSPI_CHCONF_FFET;
328	xferlevel |= bytes_per_word - 1;
329	}
330
331	mcspi_write_reg(ctlr, OMAP2_MCSPI_XFERLEVEL, val: xferlevel);
332	mcspi_write_chconf0(spi, val: chconf);
333	mcspi->fifo_depth = max_fifo_depth;
334
335	return;
336	}
337
338	disable_fifo:
339	if (t->rx_buf != NULL)
340	chconf &= ~OMAP2_MCSPI_CHCONF_FFER;
341
342	if (t->tx_buf != NULL)
343	chconf &= ~OMAP2_MCSPI_CHCONF_FFET;
344
345	mcspi_write_chconf0(spi, val: chconf);
346	mcspi->fifo_depth = 0;
347	}
348
349	static int mcspi_wait_for_reg_bit(void __iomem *reg, unsigned long bit)
350	{
351	unsigned long timeout;
352
353	timeout = jiffies + msecs_to_jiffies(m: 1000);
354	while (!(readl_relaxed(reg) & bit)) {
355	if (time_after(jiffies, timeout)) {
356	if (!(readl_relaxed(reg) & bit))
357	return -ETIMEDOUT;
358	else
359	return 0;
360	}
361	cpu_relax();
362	}
363	return 0;
364	}
365
366	static int mcspi_wait_for_completion(struct omap2_mcspi *mcspi,
367	struct completion *x)
368	{
369	if (spi_controller_is_target(ctlr: mcspi->ctlr)) {
370	if (wait_for_completion_interruptible(x) ||
371	mcspi->target_aborted)
372	return -EINTR;
373	} else {
374	wait_for_completion(x);
375	}
376
377	return 0;
378	}
379
380	static void omap2_mcspi_rx_callback(void *data)
381	{
382	struct spi_device *spi = data;
383	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr: spi->controller);
384	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
385
386	/* We must disable the DMA RX request */
387	omap2_mcspi_set_dma_req(spi, is_read: 1, enable: 0);
388
389	complete(&mcspi_dma->dma_rx_completion);
390	}
391
392	static void omap2_mcspi_tx_callback(void *data)
393	{
394	struct spi_device *spi = data;
395	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr: spi->controller);
396	struct omap2_mcspi_dma *mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
397
398	/* We must disable the DMA TX request */
399	omap2_mcspi_set_dma_req(spi, is_read: 0, enable: 0);
400
401	complete(&mcspi_dma->dma_tx_completion);
402	}
403
404	static void omap2_mcspi_tx_dma(struct spi_device *spi,
405	struct spi_transfer *xfer,
406	struct dma_slave_config cfg)
407	{
408	struct omap2_mcspi *mcspi;
409	struct omap2_mcspi_dma *mcspi_dma;
410	struct dma_async_tx_descriptor *tx;
411
412	mcspi = spi_controller_get_devdata(ctlr: spi->controller);
413	mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
414
415	dmaengine_slave_config(chan: mcspi_dma->dma_tx, config: &cfg);
416
417	tx = dmaengine_prep_slave_sg(chan: mcspi_dma->dma_tx, sgl: xfer->tx_sg.sgl,
418	sg_len: xfer->tx_sg.nents,
419	dir: DMA_MEM_TO_DEV,
420	flags: DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
421	if (tx) {
422	tx->callback = omap2_mcspi_tx_callback;
423	tx->callback_param = spi;
424	dmaengine_submit(desc: tx);
425	} else {
426	/* FIXME: fall back to PIO? */
427	}
428	dma_async_issue_pending(chan: mcspi_dma->dma_tx);
429	omap2_mcspi_set_dma_req(spi, is_read: 0, enable: 1);
430	}
431
432	static unsigned
433	omap2_mcspi_rx_dma(struct spi_device *spi, struct spi_transfer *xfer,
434	struct dma_slave_config cfg,
435	unsigned es)
436	{
437	struct omap2_mcspi *mcspi;
438	struct omap2_mcspi_dma *mcspi_dma;
439	unsigned int count, transfer_reduction = 0;
440	struct scatterlist *sg_out[2];
441	int nb_sizes = 0, out_mapped_nents[2], ret, x;
442	size_t sizes[2];
443	u32 l;
444	int elements = 0;
445	int word_len, element_count;
446	struct omap2_mcspi_cs *cs = spi->controller_state;
447	void __iomem *chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
448	struct dma_async_tx_descriptor *tx;
449
450	mcspi = spi_controller_get_devdata(ctlr: spi->controller);
451	mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
452	count = xfer->len;
453
454	/*
455	* In the "End-of-Transfer Procedure" section for DMA RX in OMAP35x TRM
456	* it mentions reducing DMA transfer length by one element in host
457	* normal mode.
458	*/
459	if (mcspi->fifo_depth == 0)
460	transfer_reduction = es;
461
462	word_len = cs->word_len;
463	l = mcspi_cached_chconf0(spi);
464
465	if (word_len <= 8)
466	element_count = count;
467	else if (word_len <= 16)
468	element_count = count >> 1;
469	else /* word_len <= 32 */
470	element_count = count >> 2;
471
472
473	dmaengine_slave_config(chan: mcspi_dma->dma_rx, config: &cfg);
474
475	/*
476	* Reduce DMA transfer length by one more if McSPI is
477	* configured in turbo mode.
478	*/
479	if ((l & OMAP2_MCSPI_CHCONF_TURBO) && mcspi->fifo_depth == 0)
480	transfer_reduction += es;
481
482	if (transfer_reduction) {
483	/* Split sgl into two. The second sgl won't be used. */
484	sizes[0] = count - transfer_reduction;
485	sizes[1] = transfer_reduction;
486	nb_sizes = 2;
487	} else {
488	/*
489	* Don't bother splitting the sgl. This essentially
490	* clones the original sgl.
491	*/
492	sizes[0] = count;
493	nb_sizes = 1;
494	}
495
496	ret = sg_split(in: xfer->rx_sg.sgl, in_mapped_nents: xfer->rx_sg.nents, skip: 0, nb_splits: nb_sizes,
497	split_sizes: sizes, out: sg_out, out_mapped_nents, GFP_KERNEL);
498
499	if (ret < 0) {
500	dev_err(&spi->dev, "sg_split failed\n");
501	return 0;
502	}
503
504	tx = dmaengine_prep_slave_sg(chan: mcspi_dma->dma_rx, sgl: sg_out[0],
505	sg_len: out_mapped_nents[0], dir: DMA_DEV_TO_MEM,
506	flags: DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
507	if (tx) {
508	tx->callback = omap2_mcspi_rx_callback;
509	tx->callback_param = spi;
510	dmaengine_submit(desc: tx);
511	} else {
512	/* FIXME: fall back to PIO? */
513	}
514
515	dma_async_issue_pending(chan: mcspi_dma->dma_rx);
516	omap2_mcspi_set_dma_req(spi, is_read: 1, enable: 1);
517
518	ret = mcspi_wait_for_completion(mcspi, x: &mcspi_dma->dma_rx_completion);
519	if (ret || mcspi->target_aborted) {
520	dmaengine_terminate_sync(chan: mcspi_dma->dma_rx);
521	omap2_mcspi_set_dma_req(spi, is_read: 1, enable: 0);
522	return 0;
523	}
524
525	for (x = 0; x < nb_sizes; x++)
526	kfree(objp: sg_out[x]);
527
528	if (mcspi->fifo_depth > 0)
529	return count;
530
531	/*
532	* Due to the DMA transfer length reduction the missing bytes must
533	* be read manually to receive all of the expected data.
534	*/
535	omap2_mcspi_set_enable(spi, enable: 0);
536
537	elements = element_count - 1;
538
539	if (l & OMAP2_MCSPI_CHCONF_TURBO) {
540	elements--;
541
542	if (!mcspi_wait_for_reg_bit(reg: chstat_reg,
543	OMAP2_MCSPI_CHSTAT_RXS)) {
544	u32 w;
545
546	w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
547	if (word_len <= 8)
548	((u8 *)xfer->rx_buf)[elements++] = w;
549	else if (word_len <= 16)
550	((u16 *)xfer->rx_buf)[elements++] = w;
551	else /* word_len <= 32 */
552	((u32 *)xfer->rx_buf)[elements++] = w;
553	} else {
554	int bytes_per_word = mcspi_bytes_per_word(word_len);
555	dev_err(&spi->dev, "DMA RX penultimate word empty\n");
556	count -= (bytes_per_word << 1);
557	omap2_mcspi_set_enable(spi, enable: 1);
558	return count;
559	}
560	}
561	if (!mcspi_wait_for_reg_bit(reg: chstat_reg, OMAP2_MCSPI_CHSTAT_RXS)) {
562	u32 w;
563
564	w = mcspi_read_cs_reg(spi, OMAP2_MCSPI_RX0);
565	if (word_len <= 8)
566	((u8 *)xfer->rx_buf)[elements] = w;
567	else if (word_len <= 16)
568	((u16 *)xfer->rx_buf)[elements] = w;
569	else /* word_len <= 32 */
570	((u32 *)xfer->rx_buf)[elements] = w;
571	} else {
572	dev_err(&spi->dev, "DMA RX last word empty\n");
573	count -= mcspi_bytes_per_word(word_len);
574	}
575	omap2_mcspi_set_enable(spi, enable: 1);
576	return count;
577	}
578
579	static unsigned
580	omap2_mcspi_txrx_dma(struct spi_device *spi, struct spi_transfer *xfer)
581	{
582	struct omap2_mcspi *mcspi;
583	struct omap2_mcspi_cs *cs = spi->controller_state;
584	struct omap2_mcspi_dma *mcspi_dma;
585	unsigned int count;
586	u8 *rx;
587	const u8 *tx;
588	struct dma_slave_config cfg;
589	enum dma_slave_buswidth width;
590	unsigned es;
591	void __iomem *chstat_reg;
592	void __iomem *irqstat_reg;
593	int wait_res;
594
595	mcspi = spi_controller_get_devdata(ctlr: spi->controller);
596	mcspi_dma = &mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
597
598	if (cs->word_len <= 8) {
599	width = DMA_SLAVE_BUSWIDTH_1_BYTE;
600	es = 1;
601	} else if (cs->word_len <= 16) {
602	width = DMA_SLAVE_BUSWIDTH_2_BYTES;
603	es = 2;
604	} else {
605	width = DMA_SLAVE_BUSWIDTH_4_BYTES;
606	es = 4;
607	}
608
609	count = xfer->len;
610
611	memset(&cfg, 0, sizeof(cfg));
612	cfg.src_addr = cs->phys + OMAP2_MCSPI_RX0;
613	cfg.dst_addr = cs->phys + OMAP2_MCSPI_TX0;
614	cfg.src_addr_width = width;
615	cfg.dst_addr_width = width;
616	cfg.src_maxburst = 1;
617	cfg.dst_maxburst = 1;
618
619	rx = xfer->rx_buf;
620	tx = xfer->tx_buf;
621
622	mcspi->target_aborted = false;
623	reinit_completion(x: &mcspi_dma->dma_tx_completion);
624	reinit_completion(x: &mcspi_dma->dma_rx_completion);
625	reinit_completion(x: &mcspi->txdone);
626	if (tx) {
627	/* Enable EOW IRQ to know end of tx in target mode */
628	if (spi_controller_is_target(ctlr: spi->controller))
629	mcspi_write_reg(ctlr: spi->controller,
630	OMAP2_MCSPI_IRQENABLE,
631	OMAP2_MCSPI_IRQSTATUS_EOW);
632	omap2_mcspi_tx_dma(spi, xfer, cfg);
633	}
634
635	if (rx != NULL)
636	count = omap2_mcspi_rx_dma(spi, xfer, cfg, es);
637
638	if (tx != NULL) {
639	int ret;
640
641	ret = mcspi_wait_for_completion(mcspi, x: &mcspi_dma->dma_tx_completion);
642	if (ret || mcspi->target_aborted) {
643	dmaengine_terminate_sync(chan: mcspi_dma->dma_tx);
644	omap2_mcspi_set_dma_req(spi, is_read: 0, enable: 0);
645	return 0;
646	}
647
648	if (spi_controller_is_target(ctlr: mcspi->ctlr)) {
649	ret = mcspi_wait_for_completion(mcspi, x: &mcspi->txdone);
650	if (ret || mcspi->target_aborted)
651	return 0;
652	}
653
654	if (mcspi->fifo_depth > 0) {
655	irqstat_reg = mcspi->base + OMAP2_MCSPI_IRQSTATUS;
656
657	if (mcspi_wait_for_reg_bit(reg: irqstat_reg,
658	OMAP2_MCSPI_IRQSTATUS_EOW) < 0)
659	dev_err(&spi->dev, "EOW timed out\n");
660
661	mcspi_write_reg(ctlr: mcspi->ctlr, OMAP2_MCSPI_IRQSTATUS,
662	OMAP2_MCSPI_IRQSTATUS_EOW);
663	}
664
665	/* for TX_ONLY mode, be sure all words have shifted out */
666	if (rx == NULL) {
667	chstat_reg = cs->base + OMAP2_MCSPI_CHSTAT0;
668	if (mcspi->fifo_depth > 0) {
669	wait_res = mcspi_wait_for_reg_bit(reg: chstat_reg,
670	OMAP2_MCSPI_CHSTAT_TXFFE);
671	if (wait_res < 0)
672	dev_err(&spi->dev, "TXFFE timed out\n");
673	} else {
674	wait_res = mcspi_wait_for_reg_bit(reg: chstat_reg,
675	OMAP2_MCSPI_CHSTAT_TXS);
676	if (wait_res < 0)
677	dev_err(&spi->dev, "TXS timed out\n");
678	}
679	if (wait_res >= 0 &&
680	(mcspi_wait_for_reg_bit(reg: chstat_reg,
681	OMAP2_MCSPI_CHSTAT_EOT) < 0))
682	dev_err(&spi->dev, "EOT timed out\n");
683	}
684	}
685	return count;
686	}
687
688	static unsigned
689	omap2_mcspi_txrx_pio(struct spi_device *spi, struct spi_transfer *xfer)
690	{
691	struct omap2_mcspi_cs *cs = spi->controller_state;
692	unsigned int count, c;
693	u32 l;
694	void __iomem *base = cs->base;
695	void __iomem *tx_reg;
696	void __iomem *rx_reg;
697	void __iomem *chstat_reg;
698	int word_len;
699
700	count = xfer->len;
701	c = count;
702	word_len = cs->word_len;
703
704	l = mcspi_cached_chconf0(spi);
705
706	/* We store the pre-calculated register addresses on stack to speed
707	* up the transfer loop. */
708	tx_reg = base + OMAP2_MCSPI_TX0;
709	rx_reg = base + OMAP2_MCSPI_RX0;
710	chstat_reg = base + OMAP2_MCSPI_CHSTAT0;
711
712	if (c < (word_len>>3))
713	return 0;
714
715	if (word_len <= 8) {
716	u8 *rx;
717	const u8 *tx;
718
719	rx = xfer->rx_buf;
720	tx = xfer->tx_buf;
721
722	do {
723	c -= 1;
724	if (tx != NULL) {
725	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
726	OMAP2_MCSPI_CHSTAT_TXS) < 0) {
727	dev_err(&spi->dev, "TXS timed out\n");
728	goto out;
729	}
730	dev_vdbg(&spi->dev, "write-%d %02x\n",
731	word_len, *tx);
732	writel_relaxed(*tx++, tx_reg);
733	}
734	if (rx != NULL) {
735	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
736	OMAP2_MCSPI_CHSTAT_RXS) < 0) {
737	dev_err(&spi->dev, "RXS timed out\n");
738	goto out;
739	}
740
741	if (c == 1 && tx == NULL &&
742	(l & OMAP2_MCSPI_CHCONF_TURBO)) {
743	omap2_mcspi_set_enable(spi, enable: 0);
744	*rx++ = readl_relaxed(rx_reg);
745	dev_vdbg(&spi->dev, "read-%d %02x\n",
746	word_len, *(rx - 1));
747	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
748	OMAP2_MCSPI_CHSTAT_RXS) < 0) {
749	dev_err(&spi->dev,
750	"RXS timed out\n");
751	goto out;
752	}
753	c = 0;
754	} else if (c == 0 && tx == NULL) {
755	omap2_mcspi_set_enable(spi, enable: 0);
756	}
757
758	*rx++ = readl_relaxed(rx_reg);
759	dev_vdbg(&spi->dev, "read-%d %02x\n",
760	word_len, *(rx - 1));
761	}
762	/* Add word delay between each word */
763	spi_delay_exec(delay: &xfer->word_delay, xfer);
764	} while (c);
765	} else if (word_len <= 16) {
766	u16 *rx;
767	const u16 *tx;
768
769	rx = xfer->rx_buf;
770	tx = xfer->tx_buf;
771	do {
772	c -= 2;
773	if (tx != NULL) {
774	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
775	OMAP2_MCSPI_CHSTAT_TXS) < 0) {
776	dev_err(&spi->dev, "TXS timed out\n");
777	goto out;
778	}
779	dev_vdbg(&spi->dev, "write-%d %04x\n",
780	word_len, *tx);
781	writel_relaxed(*tx++, tx_reg);
782	}
783	if (rx != NULL) {
784	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
785	OMAP2_MCSPI_CHSTAT_RXS) < 0) {
786	dev_err(&spi->dev, "RXS timed out\n");
787	goto out;
788	}
789
790	if (c == 2 && tx == NULL &&
791	(l & OMAP2_MCSPI_CHCONF_TURBO)) {
792	omap2_mcspi_set_enable(spi, enable: 0);
793	*rx++ = readl_relaxed(rx_reg);
794	dev_vdbg(&spi->dev, "read-%d %04x\n",
795	word_len, *(rx - 1));
796	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
797	OMAP2_MCSPI_CHSTAT_RXS) < 0) {
798	dev_err(&spi->dev,
799	"RXS timed out\n");
800	goto out;
801	}
802	c = 0;
803	} else if (c == 0 && tx == NULL) {
804	omap2_mcspi_set_enable(spi, enable: 0);
805	}
806
807	*rx++ = readl_relaxed(rx_reg);
808	dev_vdbg(&spi->dev, "read-%d %04x\n",
809	word_len, *(rx - 1));
810	}
811	/* Add word delay between each word */
812	spi_delay_exec(delay: &xfer->word_delay, xfer);
813	} while (c >= 2);
814	} else if (word_len <= 32) {
815	u32 *rx;
816	const u32 *tx;
817
818	rx = xfer->rx_buf;
819	tx = xfer->tx_buf;
820	do {
821	c -= 4;
822	if (tx != NULL) {
823	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
824	OMAP2_MCSPI_CHSTAT_TXS) < 0) {
825	dev_err(&spi->dev, "TXS timed out\n");
826	goto out;
827	}
828	dev_vdbg(&spi->dev, "write-%d %08x\n",
829	word_len, *tx);
830	writel_relaxed(*tx++, tx_reg);
831	}
832	if (rx != NULL) {
833	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
834	OMAP2_MCSPI_CHSTAT_RXS) < 0) {
835	dev_err(&spi->dev, "RXS timed out\n");
836	goto out;
837	}
838
839	if (c == 4 && tx == NULL &&
840	(l & OMAP2_MCSPI_CHCONF_TURBO)) {
841	omap2_mcspi_set_enable(spi, enable: 0);
842	*rx++ = readl_relaxed(rx_reg);
843	dev_vdbg(&spi->dev, "read-%d %08x\n",
844	word_len, *(rx - 1));
845	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
846	OMAP2_MCSPI_CHSTAT_RXS) < 0) {
847	dev_err(&spi->dev,
848	"RXS timed out\n");
849	goto out;
850	}
851	c = 0;
852	} else if (c == 0 && tx == NULL) {
853	omap2_mcspi_set_enable(spi, enable: 0);
854	}
855
856	*rx++ = readl_relaxed(rx_reg);
857	dev_vdbg(&spi->dev, "read-%d %08x\n",
858	word_len, *(rx - 1));
859	}
860	/* Add word delay between each word */
861	spi_delay_exec(delay: &xfer->word_delay, xfer);
862	} while (c >= 4);
863	}
864
865	/* for TX_ONLY mode, be sure all words have shifted out */
866	if (xfer->rx_buf == NULL) {
867	if (mcspi_wait_for_reg_bit(reg: chstat_reg,
868	OMAP2_MCSPI_CHSTAT_TXS) < 0) {
869	dev_err(&spi->dev, "TXS timed out\n");
870	} else if (mcspi_wait_for_reg_bit(reg: chstat_reg,
871	OMAP2_MCSPI_CHSTAT_EOT) < 0)
872	dev_err(&spi->dev, "EOT timed out\n");
873
874	/* disable chan to purge rx datas received in TX_ONLY transfer,
875	* otherwise these rx datas will affect the direct following
876	* RX_ONLY transfer.
877	*/
878	omap2_mcspi_set_enable(spi, enable: 0);
879	}
880	out:
881	omap2_mcspi_set_enable(spi, enable: 1);
882	return count - c;
883	}
884
885	static u32 omap2_mcspi_calc_divisor(u32 speed_hz, u32 ref_clk_hz)
886	{
887	u32 div;
888
889	for (div = 0; div < 15; div++)
890	if (speed_hz >= (ref_clk_hz >> div))
891	return div;
892
893	return 15;
894	}
895
896	/* called only when no transfer is active to this device */
897	static int omap2_mcspi_setup_transfer(struct spi_device *spi,
898	struct spi_transfer *t)
899	{
900	struct omap2_mcspi_cs *cs = spi->controller_state;
901	struct omap2_mcspi *mcspi;
902	u32 ref_clk_hz, l = 0, clkd = 0, div, extclk = 0, clkg = 0;
903	u8 word_len = spi->bits_per_word;
904	u32 speed_hz = spi->max_speed_hz;
905
906	mcspi = spi_controller_get_devdata(ctlr: spi->controller);
907
908	if (t != NULL && t->bits_per_word)
909	word_len = t->bits_per_word;
910
911	cs->word_len = word_len;
912
913	if (t && t->speed_hz)
914	speed_hz = t->speed_hz;
915
916	ref_clk_hz = mcspi->ref_clk_hz;
917	speed_hz = min_t(u32, speed_hz, ref_clk_hz);
918	if (speed_hz < (ref_clk_hz / OMAP2_MCSPI_MAX_DIVIDER)) {
919	clkd = omap2_mcspi_calc_divisor(speed_hz, ref_clk_hz);
920	speed_hz = ref_clk_hz >> clkd;
921	clkg = 0;
922	} else {
923	div = (ref_clk_hz + speed_hz - 1) / speed_hz;
924	speed_hz = ref_clk_hz / div;
925	clkd = (div - 1) & 0xf;
926	extclk = (div - 1) >> 4;
927	clkg = OMAP2_MCSPI_CHCONF_CLKG;
928	}
929
930	l = mcspi_cached_chconf0(spi);
931
932	/* standard 4-wire host mode: SCK, MOSI/out, MISO/in, nCS
933	* REVISIT: this controller could support SPI_3WIRE mode.
934	*/
935	if (mcspi->pin_dir == MCSPI_PINDIR_D0_IN_D1_OUT) {
936	l &= ~OMAP2_MCSPI_CHCONF_IS;
937	l &= ~OMAP2_MCSPI_CHCONF_DPE1;
938	l |= OMAP2_MCSPI_CHCONF_DPE0;
939	} else {
940	l |= OMAP2_MCSPI_CHCONF_IS;
941	l |= OMAP2_MCSPI_CHCONF_DPE1;
942	l &= ~OMAP2_MCSPI_CHCONF_DPE0;
943	}
944
945	/* wordlength */
946	l &= ~OMAP2_MCSPI_CHCONF_WL_MASK;
947	l |= (word_len - 1) << 7;
948
949	/* set chipselect polarity; manage with FORCE */
950	if (!(spi->mode & SPI_CS_HIGH))
951	l |= OMAP2_MCSPI_CHCONF_EPOL; /* active-low; normal */
952	else
953	l &= ~OMAP2_MCSPI_CHCONF_EPOL;
954
955	/* set clock divisor */
956	l &= ~OMAP2_MCSPI_CHCONF_CLKD_MASK;
957	l |= clkd << 2;
958
959	/* set clock granularity */
960	l &= ~OMAP2_MCSPI_CHCONF_CLKG;
961	l |= clkg;
962	if (clkg) {
963	cs->chctrl0 &= ~OMAP2_MCSPI_CHCTRL_EXTCLK_MASK;
964	cs->chctrl0 |= extclk << 8;
965	mcspi_write_cs_reg(spi, OMAP2_MCSPI_CHCTRL0, val: cs->chctrl0);
966	}
967
968	/* set SPI mode 0..3 */
969	if (spi->mode & SPI_CPOL)
970	l |= OMAP2_MCSPI_CHCONF_POL;
971	else
972	l &= ~OMAP2_MCSPI_CHCONF_POL;
973	if (spi->mode & SPI_CPHA)
974	l |= OMAP2_MCSPI_CHCONF_PHA;
975	else
976	l &= ~OMAP2_MCSPI_CHCONF_PHA;
977
978	mcspi_write_chconf0(spi, val: l);
979
980	cs->mode = spi->mode;
981
982	dev_dbg(&spi->dev, "setup: speed %d, sample %s edge, clk %s\n",
983	speed_hz,
984	(spi->mode & SPI_CPHA) ? "trailing" : "leading",
985	(spi->mode & SPI_CPOL) ? "inverted" : "normal");
986
987	return 0;
988	}
989
990	/*
991	* Note that we currently allow DMA only if we get a channel
992	* for both rx and tx. Otherwise we'll do PIO for both rx and tx.
993	*/
994	static int omap2_mcspi_request_dma(struct omap2_mcspi *mcspi,
995	struct omap2_mcspi_dma *mcspi_dma)
996	{
997	int ret = 0;
998
999	mcspi_dma->dma_rx = dma_request_chan(dev: mcspi->dev,
1000	name: mcspi_dma->dma_rx_ch_name);
1001	if (IS_ERR(ptr: mcspi_dma->dma_rx)) {
1002	ret = PTR_ERR(ptr: mcspi_dma->dma_rx);
1003	mcspi_dma->dma_rx = NULL;
1004	goto no_dma;
1005	}
1006
1007	mcspi_dma->dma_tx = dma_request_chan(dev: mcspi->dev,
1008	name: mcspi_dma->dma_tx_ch_name);
1009	if (IS_ERR(ptr: mcspi_dma->dma_tx)) {
1010	ret = PTR_ERR(ptr: mcspi_dma->dma_tx);
1011	mcspi_dma->dma_tx = NULL;
1012	dma_release_channel(chan: mcspi_dma->dma_rx);
1013	mcspi_dma->dma_rx = NULL;
1014	}
1015
1016	init_completion(x: &mcspi_dma->dma_rx_completion);
1017	init_completion(x: &mcspi_dma->dma_tx_completion);
1018
1019	no_dma:
1020	return ret;
1021	}
1022
1023	static void omap2_mcspi_release_dma(struct spi_controller *ctlr)
1024	{
1025	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1026	struct omap2_mcspi_dma *mcspi_dma;
1027	int i;
1028
1029	for (i = 0; i < ctlr->num_chipselect; i++) {
1030	mcspi_dma = &mcspi->dma_channels[i];
1031
1032	if (mcspi_dma->dma_rx) {
1033	dma_release_channel(chan: mcspi_dma->dma_rx);
1034	mcspi_dma->dma_rx = NULL;
1035	}
1036	if (mcspi_dma->dma_tx) {
1037	dma_release_channel(chan: mcspi_dma->dma_tx);
1038	mcspi_dma->dma_tx = NULL;
1039	}
1040	}
1041	}
1042
1043	static void omap2_mcspi_cleanup(struct spi_device *spi)
1044	{
1045	struct omap2_mcspi_cs *cs;
1046
1047	if (spi->controller_state) {
1048	/* Unlink controller state from context save list */
1049	cs = spi->controller_state;
1050	list_del(entry: &cs->node);
1051
1052	kfree(objp: cs);
1053	}
1054	}
1055
1056	static int omap2_mcspi_setup(struct spi_device *spi)
1057	{
1058	bool initial_setup = false;
1059	int ret;
1060	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr: spi->controller);
1061	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1062	struct omap2_mcspi_cs *cs = spi->controller_state;
1063
1064	if (!cs) {
1065	cs = kzalloc(size: sizeof(*cs), GFP_KERNEL);
1066	if (!cs)
1067	return -ENOMEM;
1068	cs->base = mcspi->base + spi_get_chipselect(spi, idx: 0) * 0x14;
1069	cs->phys = mcspi->phys + spi_get_chipselect(spi, idx: 0) * 0x14;
1070	cs->mode = 0;
1071	cs->chconf0 = 0;
1072	cs->chctrl0 = 0;
1073	spi->controller_state = cs;
1074	/* Link this to context save list */
1075	list_add_tail(new: &cs->node, head: &ctx->cs);
1076	initial_setup = true;
1077	}
1078
1079	ret = pm_runtime_resume_and_get(dev: mcspi->dev);
1080	if (ret < 0) {
1081	if (initial_setup)
1082	omap2_mcspi_cleanup(spi);
1083
1084	return ret;
1085	}
1086
1087	ret = omap2_mcspi_setup_transfer(spi, NULL);
1088	if (ret && initial_setup)
1089	omap2_mcspi_cleanup(spi);
1090
1091	pm_runtime_mark_last_busy(dev: mcspi->dev);
1092	pm_runtime_put_autosuspend(dev: mcspi->dev);
1093
1094	return ret;
1095	}
1096
1097	static irqreturn_t omap2_mcspi_irq_handler(int irq, void *data)
1098	{
1099	struct omap2_mcspi *mcspi = data;
1100	u32 irqstat;
1101
1102	irqstat = mcspi_read_reg(ctlr: mcspi->ctlr, OMAP2_MCSPI_IRQSTATUS);
1103	if (!irqstat)
1104	return IRQ_NONE;
1105
1106	/* Disable IRQ and wakeup target xfer task */
1107	mcspi_write_reg(ctlr: mcspi->ctlr, OMAP2_MCSPI_IRQENABLE, val: 0);
1108	if (irqstat & OMAP2_MCSPI_IRQSTATUS_EOW)
1109	complete(&mcspi->txdone);
1110
1111	return IRQ_HANDLED;
1112	}
1113
1114	static int omap2_mcspi_target_abort(struct spi_controller *ctlr)
1115	{
1116	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1117	struct omap2_mcspi_dma *mcspi_dma = mcspi->dma_channels;
1118
1119	mcspi->target_aborted = true;
1120	complete(&mcspi_dma->dma_rx_completion);
1121	complete(&mcspi_dma->dma_tx_completion);
1122	complete(&mcspi->txdone);
1123
1124	return 0;
1125	}
1126
1127	static int omap2_mcspi_transfer_one(struct spi_controller *ctlr,
1128	struct spi_device *spi,
1129	struct spi_transfer *t)
1130	{
1131
1132	/* We only enable one channel at a time -- the one whose message is
1133	* -- although this controller would gladly
1134	* arbitrate among multiple channels. This corresponds to "single
1135	* channel" host mode. As a side effect, we need to manage the
1136	* chipselect with the FORCE bit ... CS != channel enable.
1137	*/
1138
1139	struct omap2_mcspi *mcspi;
1140	struct omap2_mcspi_dma *mcspi_dma;
1141	struct omap2_mcspi_cs *cs;
1142	struct omap2_mcspi_device_config *cd;
1143	int par_override = 0;
1144	int status = 0;
1145	u32 chconf;
1146
1147	mcspi = spi_controller_get_devdata(ctlr);
1148	mcspi_dma = mcspi->dma_channels + spi_get_chipselect(spi, idx: 0);
1149	cs = spi->controller_state;
1150	cd = spi->controller_data;
1151
1152	/*
1153	* The target driver could have changed spi->mode in which case
1154	* it will be different from cs->mode (the current hardware setup).
1155	* If so, set par_override (even though its not a parity issue) so
1156	* omap2_mcspi_setup_transfer will be called to configure the hardware
1157	* with the correct mode on the first iteration of the loop below.
1158	*/
1159	if (spi->mode != cs->mode)
1160	par_override = 1;
1161
1162	omap2_mcspi_set_enable(spi, enable: 0);
1163
1164	if (spi_get_csgpiod(spi, idx: 0))
1165	omap2_mcspi_set_cs(spi, enable: spi->mode & SPI_CS_HIGH);
1166
1167	if (par_override ||
1168	(t->speed_hz != spi->max_speed_hz) ||
1169	(t->bits_per_word != spi->bits_per_word)) {
1170	par_override = 1;
1171	status = omap2_mcspi_setup_transfer(spi, t);
1172	if (status < 0)
1173	goto out;
1174	if (t->speed_hz == spi->max_speed_hz &&
1175	t->bits_per_word == spi->bits_per_word)
1176	par_override = 0;
1177	}
1178	if (cd && cd->cs_per_word) {
1179	chconf = mcspi->ctx.modulctrl;
1180	chconf &= ~OMAP2_MCSPI_MODULCTRL_SINGLE;
1181	mcspi_write_reg(ctlr, OMAP2_MCSPI_MODULCTRL, val: chconf);
1182	mcspi->ctx.modulctrl =
1183	mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1184	}
1185
1186	chconf = mcspi_cached_chconf0(spi);
1187	chconf &= ~OMAP2_MCSPI_CHCONF_TRM_MASK;
1188	chconf &= ~OMAP2_MCSPI_CHCONF_TURBO;
1189
1190	if (t->tx_buf == NULL)
1191	chconf |= OMAP2_MCSPI_CHCONF_TRM_RX_ONLY;
1192	else if (t->rx_buf == NULL)
1193	chconf |= OMAP2_MCSPI_CHCONF_TRM_TX_ONLY;
1194
1195	if (cd && cd->turbo_mode && t->tx_buf == NULL) {
1196	/* Turbo mode is for more than one word */
1197	if (t->len > ((cs->word_len + 7) >> 3))
1198	chconf |= OMAP2_MCSPI_CHCONF_TURBO;
1199	}
1200
1201	mcspi_write_chconf0(spi, val: chconf);
1202
1203	if (t->len) {
1204	unsigned count;
1205
1206	if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1207	ctlr->cur_msg_mapped &&
1208	ctlr->can_dma(ctlr, spi, t))
1209	omap2_mcspi_set_fifo(spi, t, enable: 1);
1210
1211	omap2_mcspi_set_enable(spi, enable: 1);
1212
1213	/* RX_ONLY mode needs dummy data in TX reg */
1214	if (t->tx_buf == NULL)
1215	writel_relaxed(0, cs->base
1216	+ OMAP2_MCSPI_TX0);
1217
1218	if ((mcspi_dma->dma_rx && mcspi_dma->dma_tx) &&
1219	ctlr->cur_msg_mapped &&
1220	ctlr->can_dma(ctlr, spi, t))
1221	count = omap2_mcspi_txrx_dma(spi, xfer: t);
1222	else
1223	count = omap2_mcspi_txrx_pio(spi, xfer: t);
1224
1225	if (count != t->len) {
1226	status = -EIO;
1227	goto out;
1228	}
1229	}
1230
1231	omap2_mcspi_set_enable(spi, enable: 0);
1232
1233	if (mcspi->fifo_depth > 0)
1234	omap2_mcspi_set_fifo(spi, t, enable: 0);
1235
1236	out:
1237	/* Restore defaults if they were overriden */
1238	if (par_override) {
1239	par_override = 0;
1240	status = omap2_mcspi_setup_transfer(spi, NULL);
1241	}
1242
1243	if (cd && cd->cs_per_word) {
1244	chconf = mcspi->ctx.modulctrl;
1245	chconf |= OMAP2_MCSPI_MODULCTRL_SINGLE;
1246	mcspi_write_reg(ctlr, OMAP2_MCSPI_MODULCTRL, val: chconf);
1247	mcspi->ctx.modulctrl =
1248	mcspi_read_cs_reg(spi, OMAP2_MCSPI_MODULCTRL);
1249	}
1250
1251	omap2_mcspi_set_enable(spi, enable: 0);
1252
1253	if (spi_get_csgpiod(spi, idx: 0))
1254	omap2_mcspi_set_cs(spi, enable: !(spi->mode & SPI_CS_HIGH));
1255
1256	if (mcspi->fifo_depth > 0 && t)
1257	omap2_mcspi_set_fifo(spi, t, enable: 0);
1258
1259	return status;
1260	}
1261
1262	static int omap2_mcspi_prepare_message(struct spi_controller *ctlr,
1263	struct spi_message *msg)
1264	{
1265	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1266	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1267	struct omap2_mcspi_cs *cs;
1268
1269	/* Only a single channel can have the FORCE bit enabled
1270	* in its chconf0 register.
1271	* Scan all channels and disable them except the current one.
1272	* A FORCE can remain from a last transfer having cs_change enabled
1273	*/
1274	list_for_each_entry(cs, &ctx->cs, node) {
1275	if (msg->spi->controller_state == cs)
1276	continue;
1277
1278	if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE)) {
1279	cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1280	writel_relaxed(cs->chconf0,
1281	cs->base + OMAP2_MCSPI_CHCONF0);
1282	readl_relaxed(cs->base + OMAP2_MCSPI_CHCONF0);
1283	}
1284	}
1285
1286	return 0;
1287	}
1288
1289	static bool omap2_mcspi_can_dma(struct spi_controller *ctlr,
1290	struct spi_device *spi,
1291	struct spi_transfer *xfer)
1292	{
1293	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr: spi->controller);
1294	struct omap2_mcspi_dma *mcspi_dma =
1295	&mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
1296
1297	if (!mcspi_dma->dma_rx || !mcspi_dma->dma_tx)
1298	return false;
1299
1300	if (spi_controller_is_target(ctlr))
1301	return true;
1302
1303	ctlr->dma_rx = mcspi_dma->dma_rx;
1304	ctlr->dma_tx = mcspi_dma->dma_tx;
1305
1306	return (xfer->len >= DMA_MIN_BYTES);
1307	}
1308
1309	static size_t omap2_mcspi_max_xfer_size(struct spi_device *spi)
1310	{
1311	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr: spi->controller);
1312	struct omap2_mcspi_dma *mcspi_dma =
1313	&mcspi->dma_channels[spi_get_chipselect(spi, idx: 0)];
1314
1315	if (mcspi->max_xfer_len && mcspi_dma->dma_rx)
1316	return mcspi->max_xfer_len;
1317
1318	return SIZE_MAX;
1319	}
1320
1321	static int omap2_mcspi_controller_setup(struct omap2_mcspi *mcspi)
1322	{
1323	struct spi_controller *ctlr = mcspi->ctlr;
1324	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1325	int ret = 0;
1326
1327	ret = pm_runtime_resume_and_get(dev: mcspi->dev);
1328	if (ret < 0)
1329	return ret;
1330
1331	mcspi_write_reg(ctlr, OMAP2_MCSPI_WAKEUPENABLE,
1332	OMAP2_MCSPI_WAKEUPENABLE_WKEN);
1333	ctx->wakeupenable = OMAP2_MCSPI_WAKEUPENABLE_WKEN;
1334
1335	omap2_mcspi_set_mode(ctlr);
1336	pm_runtime_mark_last_busy(dev: mcspi->dev);
1337	pm_runtime_put_autosuspend(dev: mcspi->dev);
1338	return 0;
1339	}
1340
1341	static int omap_mcspi_runtime_suspend(struct device *dev)
1342	{
1343	int error;
1344
1345	error = pinctrl_pm_select_idle_state(dev);
1346	if (error)
1347	dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1348
1349	return 0;
1350	}
1351
1352	/*
1353	* When SPI wake up from off-mode, CS is in activate state. If it was in
1354	* inactive state when driver was suspend, then force it to inactive state at
1355	* wake up.
1356	*/
1357	static int omap_mcspi_runtime_resume(struct device *dev)
1358	{
1359	struct spi_controller *ctlr = dev_get_drvdata(dev);
1360	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1361	struct omap2_mcspi_regs *ctx = &mcspi->ctx;
1362	struct omap2_mcspi_cs *cs;
1363	int error;
1364
1365	error = pinctrl_pm_select_default_state(dev);
1366	if (error)
1367	dev_warn(dev, "%s: failed to set pins: %i\n", __func__, error);
1368
1369	/* McSPI: context restore */
1370	mcspi_write_reg(ctlr, OMAP2_MCSPI_MODULCTRL, val: ctx->modulctrl);
1371	mcspi_write_reg(ctlr, OMAP2_MCSPI_WAKEUPENABLE, val: ctx->wakeupenable);
1372
1373	list_for_each_entry(cs, &ctx->cs, node) {
1374	/*
1375	* We need to toggle CS state for OMAP take this
1376	* change in account.
1377	*/
1378	if ((cs->chconf0 & OMAP2_MCSPI_CHCONF_FORCE) == 0) {
1379	cs->chconf0 |= OMAP2_MCSPI_CHCONF_FORCE;
1380	writel_relaxed(cs->chconf0,
1381	cs->base + OMAP2_MCSPI_CHCONF0);
1382	cs->chconf0 &= ~OMAP2_MCSPI_CHCONF_FORCE;
1383	writel_relaxed(cs->chconf0,
1384	cs->base + OMAP2_MCSPI_CHCONF0);
1385	} else {
1386	writel_relaxed(cs->chconf0,
1387	cs->base + OMAP2_MCSPI_CHCONF0);
1388	}
1389	}
1390
1391	return 0;
1392	}
1393
1394	static struct omap2_mcspi_platform_config omap2_pdata = {
1395	.regs_offset = 0,
1396	};
1397
1398	static struct omap2_mcspi_platform_config omap4_pdata = {
1399	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
1400	};
1401
1402	static struct omap2_mcspi_platform_config am654_pdata = {
1403	.regs_offset = OMAP4_MCSPI_REG_OFFSET,
1404	.max_xfer_len = SZ_4K - 1,
1405	};
1406
1407	static const struct of_device_id omap_mcspi_of_match[] = {
1408	{
1409	.compatible = "ti,omap2-mcspi",
1410	.data = &omap2_pdata,
1411	},
1412	{
1413	.compatible = "ti,omap4-mcspi",
1414	.data = &omap4_pdata,
1415	},
1416	{
1417	.compatible = "ti,am654-mcspi",
1418	.data = &am654_pdata,
1419	},
1420	{ },
1421	};
1422	MODULE_DEVICE_TABLE(of, omap_mcspi_of_match);
1423
1424	static int omap2_mcspi_probe(struct platform_device *pdev)
1425	{
1426	struct spi_controller *ctlr;
1427	const struct omap2_mcspi_platform_config *pdata;
1428	struct omap2_mcspi *mcspi;
1429	struct resource *r;
1430	int status = 0, i;
1431	u32 regs_offset = 0;
1432	struct device_node *node = pdev->dev.of_node;
1433	const struct of_device_id *match;
1434
1435	if (of_property_read_bool(np: node, propname: "spi-slave"))
1436	ctlr = spi_alloc_target(dev: &pdev->dev, size: sizeof(*mcspi));
1437	else
1438	ctlr = spi_alloc_host(dev: &pdev->dev, size: sizeof(*mcspi));
1439	if (!ctlr)
1440	return -ENOMEM;
1441
1442	/* the spi->mode bits understood by this driver: */
1443	ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH;
1444	ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
1445	ctlr->setup = omap2_mcspi_setup;
1446	ctlr->auto_runtime_pm = true;
1447	ctlr->prepare_message = omap2_mcspi_prepare_message;
1448	ctlr->can_dma = omap2_mcspi_can_dma;
1449	ctlr->transfer_one = omap2_mcspi_transfer_one;
1450	ctlr->set_cs = omap2_mcspi_set_cs;
1451	ctlr->cleanup = omap2_mcspi_cleanup;
1452	ctlr->target_abort = omap2_mcspi_target_abort;
1453	ctlr->dev.of_node = node;
1454	ctlr->use_gpio_descriptors = true;
1455
1456	platform_set_drvdata(pdev, data: ctlr);
1457
1458	mcspi = spi_controller_get_devdata(ctlr);
1459	mcspi->ctlr = ctlr;
1460
1461	match = of_match_device(matches: omap_mcspi_of_match, dev: &pdev->dev);
1462	if (match) {
1463	u32 num_cs = 1; /* default number of chipselect */
1464	pdata = match->data;
1465
1466	of_property_read_u32(np: node, propname: "ti,spi-num-cs", out_value: &num_cs);
1467	ctlr->num_chipselect = num_cs;
1468	if (of_property_read_bool(np: node, propname: "ti,pindir-d0-out-d1-in"))
1469	mcspi->pin_dir = MCSPI_PINDIR_D0_OUT_D1_IN;
1470	} else {
1471	pdata = dev_get_platdata(dev: &pdev->dev);
1472	ctlr->num_chipselect = pdata->num_cs;
1473	mcspi->pin_dir = pdata->pin_dir;
1474	}
1475	regs_offset = pdata->regs_offset;
1476	if (pdata->max_xfer_len) {
1477	mcspi->max_xfer_len = pdata->max_xfer_len;
1478	ctlr->max_transfer_size = omap2_mcspi_max_xfer_size;
1479	}
1480
1481	mcspi->base = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &r);
1482	if (IS_ERR(ptr: mcspi->base)) {
1483	status = PTR_ERR(ptr: mcspi->base);
1484	goto free_ctlr;
1485	}
1486	mcspi->phys = r->start + regs_offset;
1487	mcspi->base += regs_offset;
1488
1489	mcspi->dev = &pdev->dev;
1490
1491	INIT_LIST_HEAD(list: &mcspi->ctx.cs);
1492
1493	mcspi->dma_channels = devm_kcalloc(dev: &pdev->dev, n: ctlr->num_chipselect,
1494	size: sizeof(struct omap2_mcspi_dma),
1495	GFP_KERNEL);
1496	if (mcspi->dma_channels == NULL) {
1497	status = -ENOMEM;
1498	goto free_ctlr;
1499	}
1500
1501	for (i = 0; i < ctlr->num_chipselect; i++) {
1502	sprintf(buf: mcspi->dma_channels[i].dma_rx_ch_name, fmt: "rx%d", i);
1503	sprintf(buf: mcspi->dma_channels[i].dma_tx_ch_name, fmt: "tx%d", i);
1504
1505	status = omap2_mcspi_request_dma(mcspi,
1506	mcspi_dma: &mcspi->dma_channels[i]);
1507	if (status == -EPROBE_DEFER)
1508	goto free_ctlr;
1509	}
1510
1511	status = platform_get_irq(pdev, 0);
1512	if (status < 0)
1513	goto free_ctlr;
1514	init_completion(x: &mcspi->txdone);
1515	status = devm_request_irq(dev: &pdev->dev, irq: status,
1516	handler: omap2_mcspi_irq_handler, irqflags: 0, devname: pdev->name,
1517	dev_id: mcspi);
1518	if (status) {
1519	dev_err(&pdev->dev, "Cannot request IRQ");
1520	goto free_ctlr;
1521	}
1522
1523	mcspi->ref_clk = devm_clk_get_optional_enabled(dev: &pdev->dev, NULL);
1524	if (mcspi->ref_clk)
1525	mcspi->ref_clk_hz = clk_get_rate(clk: mcspi->ref_clk);
1526	else
1527	mcspi->ref_clk_hz = OMAP2_MCSPI_MAX_FREQ;
1528	ctlr->max_speed_hz = mcspi->ref_clk_hz;
1529	ctlr->min_speed_hz = mcspi->ref_clk_hz >> 15;
1530
1531	pm_runtime_use_autosuspend(dev: &pdev->dev);
1532	pm_runtime_set_autosuspend_delay(dev: &pdev->dev, SPI_AUTOSUSPEND_TIMEOUT);
1533	pm_runtime_enable(dev: &pdev->dev);
1534
1535	status = omap2_mcspi_controller_setup(mcspi);
1536	if (status < 0)
1537	goto disable_pm;
1538
1539	status = devm_spi_register_controller(dev: &pdev->dev, ctlr);
1540	if (status < 0)
1541	goto disable_pm;
1542
1543	return status;
1544
1545	disable_pm:
1546	pm_runtime_dont_use_autosuspend(dev: &pdev->dev);
1547	pm_runtime_put_sync(dev: &pdev->dev);
1548	pm_runtime_disable(dev: &pdev->dev);
1549	free_ctlr:
1550	omap2_mcspi_release_dma(ctlr);
1551	spi_controller_put(ctlr);
1552	return status;
1553	}
1554
1555	static void omap2_mcspi_remove(struct platform_device *pdev)
1556	{
1557	struct spi_controller *ctlr = platform_get_drvdata(pdev);
1558	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1559
1560	omap2_mcspi_release_dma(ctlr);
1561
1562	pm_runtime_dont_use_autosuspend(dev: mcspi->dev);
1563	pm_runtime_put_sync(dev: mcspi->dev);
1564	pm_runtime_disable(dev: &pdev->dev);
1565	}
1566
1567	/* work with hotplug and coldplug */
1568	MODULE_ALIAS("platform:omap2_mcspi");
1569
1570	static int __maybe_unused omap2_mcspi_suspend(struct device *dev)
1571	{
1572	struct spi_controller *ctlr = dev_get_drvdata(dev);
1573	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1574	int error;
1575
1576	error = pinctrl_pm_select_sleep_state(dev);
1577	if (error)
1578	dev_warn(mcspi->dev, "%s: failed to set pins: %i\n",
1579	__func__, error);
1580
1581	error = spi_controller_suspend(ctlr);
1582	if (error)
1583	dev_warn(mcspi->dev, "%s: controller suspend failed: %i\n",
1584	__func__, error);
1585
1586	return pm_runtime_force_suspend(dev);
1587	}
1588
1589	static int __maybe_unused omap2_mcspi_resume(struct device *dev)
1590	{
1591	struct spi_controller *ctlr = dev_get_drvdata(dev);
1592	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
1593	int error;
1594
1595	error = spi_controller_resume(ctlr);
1596	if (error)
1597	dev_warn(mcspi->dev, "%s: controller resume failed: %i\n",
1598	__func__, error);
1599
1600	return pm_runtime_force_resume(dev);
1601	}
1602
1603	static const struct dev_pm_ops omap2_mcspi_pm_ops = {
1604	SET_SYSTEM_SLEEP_PM_OPS(omap2_mcspi_suspend,
1605	omap2_mcspi_resume)
1606	.runtime_suspend = omap_mcspi_runtime_suspend,
1607	.runtime_resume = omap_mcspi_runtime_resume,
1608	};
1609
1610	static struct platform_driver omap2_mcspi_driver = {
1611	.driver = {
1612	.name = "omap2_mcspi",
1613	.pm = &omap2_mcspi_pm_ops,
1614	.of_match_table = omap_mcspi_of_match,
1615	},
1616	.probe = omap2_mcspi_probe,
1617	.remove_new = omap2_mcspi_remove,
1618	};
1619
1620	module_platform_driver(omap2_mcspi_driver);
1621	MODULE_LICENSE("GPL");
1622