1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ARM Message Handling Unit Version 2 (MHUv2) driver.
4 *
5 * Copyright (C) 2020 ARM Ltd.
6 * Copyright (C) 2020 Linaro Ltd.
7 *
8 * An MHUv2 mailbox controller can provide up to 124 channel windows (each 32
9 * bit long) and the driver allows any combination of both the transport
10 * protocol modes: data-transfer and doorbell, to be used on those channel
11 * windows.
12 *
13 * The transport protocols should be specified in the device tree entry for the
14 * device. The transport protocols determine how the underlying hardware
15 * resources of the device are utilized when transmitting data. Refer to the
16 * device tree bindings of the ARM MHUv2 controller for more details.
17 *
18 * The number of registered mailbox channels is dependent on both the underlying
19 * hardware - mainly the number of channel windows implemented by the platform,
20 * as well as the selected transport protocols.
21 *
22 * The MHUv2 controller can work both as a sender and receiver, but the driver
23 * and the DT bindings support unidirectional transfers for better allocation of
24 * the channels. That is, this driver will be probed for two separate devices
25 * for each mailbox controller, a sender device and a receiver device.
26 */
27
28#include <linux/amba/bus.h>
29#include <linux/interrupt.h>
30#include <linux/mailbox_controller.h>
31#include <linux/mailbox/arm_mhuv2_message.h>
32#include <linux/module.h>
33#include <linux/of_address.h>
34#include <linux/spinlock.h>
35
36/* ====== MHUv2 Registers ====== */
37
38/* Maximum number of channel windows */
39#define MHUV2_CH_WN_MAX 124
40/* Number of combined interrupt status registers */
41#define MHUV2_CMB_INT_ST_REG_CNT 4
42#define MHUV2_STAT_BYTES (sizeof(u32))
43#define MHUV2_STAT_BITS (MHUV2_STAT_BYTES * __CHAR_BIT__)
44
45#define LSB_MASK(n) ((1 << (n * __CHAR_BIT__)) - 1)
46#define MHUV2_PROTOCOL_PROP "arm,mhuv2-protocols"
47
48/* Register Message Handling Unit Configuration fields */
49struct mhu_cfg_t {
50 u32 num_ch : 7;
51 u32 pad : 25;
52} __packed;
53
54/* register Interrupt Status fields */
55struct int_st_t {
56 u32 nr2r : 1;
57 u32 r2nr : 1;
58 u32 pad : 30;
59} __packed;
60
61/* Register Interrupt Clear fields */
62struct int_clr_t {
63 u32 nr2r : 1;
64 u32 r2nr : 1;
65 u32 pad : 30;
66} __packed;
67
68/* Register Interrupt Enable fields */
69struct int_en_t {
70 u32 r2nr : 1;
71 u32 nr2r : 1;
72 u32 chcomb : 1;
73 u32 pad : 29;
74} __packed;
75
76/* Register Implementer Identification fields */
77struct iidr_t {
78 u32 implementer : 12;
79 u32 revision : 4;
80 u32 variant : 4;
81 u32 product_id : 12;
82} __packed;
83
84/* Register Architecture Identification Register fields */
85struct aidr_t {
86 u32 arch_minor_rev : 4;
87 u32 arch_major_rev : 4;
88 u32 pad : 24;
89} __packed;
90
91/* Sender Channel Window fields */
92struct mhu2_send_ch_wn_reg {
93 u32 stat;
94 u8 pad1[0x0C - 0x04];
95 u32 stat_set;
96 u32 int_st;
97 u32 int_clr;
98 u32 int_en;
99 u8 pad2[0x20 - 0x1C];
100} __packed;
101
102/* Sender frame register fields */
103struct mhu2_send_frame_reg {
104 struct mhu2_send_ch_wn_reg ch_wn[MHUV2_CH_WN_MAX];
105 struct mhu_cfg_t mhu_cfg;
106 u32 resp_cfg;
107 u32 access_request;
108 u32 access_ready;
109 struct int_st_t int_st;
110 struct int_clr_t int_clr;
111 struct int_en_t int_en;
112 u32 reserved0;
113 u32 chcomb_int_st[MHUV2_CMB_INT_ST_REG_CNT];
114 u8 pad[0xFC8 - 0xFB0];
115 struct iidr_t iidr;
116 struct aidr_t aidr;
117} __packed;
118
119/* Receiver Channel Window fields */
120struct mhu2_recv_ch_wn_reg {
121 u32 stat;
122 u32 stat_masked;
123 u32 stat_clear;
124 u8 reserved0[0x10 - 0x0C];
125 u32 mask;
126 u32 mask_set;
127 u32 mask_clear;
128 u8 pad[0x20 - 0x1C];
129} __packed;
130
131/* Receiver frame register fields */
132struct mhu2_recv_frame_reg {
133 struct mhu2_recv_ch_wn_reg ch_wn[MHUV2_CH_WN_MAX];
134 struct mhu_cfg_t mhu_cfg;
135 u8 reserved0[0xF90 - 0xF84];
136 struct int_st_t int_st;
137 struct int_clr_t int_clr;
138 struct int_en_t int_en;
139 u32 pad;
140 u32 chcomb_int_st[MHUV2_CMB_INT_ST_REG_CNT];
141 u8 reserved2[0xFC8 - 0xFB0];
142 struct iidr_t iidr;
143 struct aidr_t aidr;
144} __packed;
145
146
147/* ====== MHUv2 data structures ====== */
148
149enum mhuv2_transport_protocol {
150 DOORBELL = 0,
151 DATA_TRANSFER = 1
152};
153
154enum mhuv2_frame {
155 RECEIVER_FRAME,
156 SENDER_FRAME
157};
158
159/**
160 * struct mhuv2 - MHUv2 mailbox controller data
161 *
162 * @mbox: Mailbox controller belonging to the MHU frame.
163 * @send: Base address of the register mapping region.
164 * @recv: Base address of the register mapping region.
165 * @frame: Frame type: RECEIVER_FRAME or SENDER_FRAME.
166 * @irq: Interrupt.
167 * @windows: Channel windows implemented by the platform.
168 * @minor: Minor version of the controller.
169 * @length: Length of the protocols array in bytes.
170 * @protocols: Raw protocol information, derived from device tree.
171 * @doorbell_pending_lock: spinlock required for correct operation of Tx
172 * interrupt for doorbells.
173 */
174struct mhuv2 {
175 struct mbox_controller mbox;
176 union {
177 struct mhu2_send_frame_reg __iomem *send;
178 struct mhu2_recv_frame_reg __iomem *recv;
179 };
180 enum mhuv2_frame frame;
181 unsigned int irq;
182 unsigned int windows;
183 unsigned int minor;
184 unsigned int length;
185 u32 *protocols;
186
187 spinlock_t doorbell_pending_lock;
188};
189
190#define mhu_from_mbox(_mbox) container_of(_mbox, struct mhuv2, mbox)
191
192/**
193 * struct mhuv2_protocol_ops - MHUv2 operations
194 *
195 * Each transport protocol must provide an implementation of the operations
196 * provided here.
197 *
198 * @rx_startup: Startup callback for receiver.
199 * @rx_shutdown: Shutdown callback for receiver.
200 * @read_data: Reads and clears newly available data.
201 * @tx_startup: Startup callback for receiver.
202 * @tx_shutdown: Shutdown callback for receiver.
203 * @last_tx_done: Report back if the last tx is completed or not.
204 * @send_data: Send data to the receiver.
205 */
206struct mhuv2_protocol_ops {
207 int (*rx_startup)(struct mhuv2 *mhu, struct mbox_chan *chan);
208 void (*rx_shutdown)(struct mhuv2 *mhu, struct mbox_chan *chan);
209 void *(*read_data)(struct mhuv2 *mhu, struct mbox_chan *chan);
210
211 void (*tx_startup)(struct mhuv2 *mhu, struct mbox_chan *chan);
212 void (*tx_shutdown)(struct mhuv2 *mhu, struct mbox_chan *chan);
213 int (*last_tx_done)(struct mhuv2 *mhu, struct mbox_chan *chan);
214 int (*send_data)(struct mhuv2 *mhu, struct mbox_chan *chan, void *arg);
215};
216
217/*
218 * MHUv2 mailbox channel's private information
219 *
220 * @ops: protocol specific ops for the channel.
221 * @ch_wn_idx: Channel window index allocated to the channel.
222 * @windows: Total number of windows consumed by the channel, only relevant
223 * in DATA_TRANSFER protocol.
224 * @doorbell: Doorbell bit number within the ch_wn_idx window, only relevant
225 * in DOORBELL protocol.
226 * @pending: Flag indicating pending doorbell interrupt, only relevant in
227 * DOORBELL protocol.
228 */
229struct mhuv2_mbox_chan_priv {
230 const struct mhuv2_protocol_ops *ops;
231 u32 ch_wn_idx;
232 union {
233 u32 windows;
234 struct {
235 u32 doorbell;
236 u32 pending;
237 };
238 };
239};
240
241/* Macro for reading a bitfield within a physically mapped packed struct */
242#define readl_relaxed_bitfield(_regptr, _type, _field) \
243 ({ \
244 u32 _regval; \
245 _regval = readl_relaxed((_regptr)); \
246 (*(_type *)(&_regval))._field; \
247 })
248
249/* Macro for writing a bitfield within a physically mapped packed struct */
250#define writel_relaxed_bitfield(_value, _regptr, _type, _field) \
251 ({ \
252 u32 _regval; \
253 _regval = readl_relaxed(_regptr); \
254 (*(_type *)(&_regval))._field = _value; \
255 writel_relaxed(_regval, _regptr); \
256 })
257
258
259/* =================== Doorbell transport protocol operations =============== */
260
261static int mhuv2_doorbell_rx_startup(struct mhuv2 *mhu, struct mbox_chan *chan)
262{
263 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
264
265 writel_relaxed(BIT(priv->doorbell),
266 &mhu->recv->ch_wn[priv->ch_wn_idx].mask_clear);
267 return 0;
268}
269
270static void mhuv2_doorbell_rx_shutdown(struct mhuv2 *mhu,
271 struct mbox_chan *chan)
272{
273 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
274
275 writel_relaxed(BIT(priv->doorbell),
276 &mhu->recv->ch_wn[priv->ch_wn_idx].mask_set);
277}
278
279static void *mhuv2_doorbell_read_data(struct mhuv2 *mhu, struct mbox_chan *chan)
280{
281 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
282
283 writel_relaxed(BIT(priv->doorbell),
284 &mhu->recv->ch_wn[priv->ch_wn_idx].stat_clear);
285 return NULL;
286}
287
288static int mhuv2_doorbell_last_tx_done(struct mhuv2 *mhu,
289 struct mbox_chan *chan)
290{
291 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
292
293 return !(readl_relaxed(&mhu->send->ch_wn[priv->ch_wn_idx].stat) &
294 BIT(priv->doorbell));
295}
296
297static int mhuv2_doorbell_send_data(struct mhuv2 *mhu, struct mbox_chan *chan,
298 void *arg)
299{
300 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
301 unsigned long flags;
302
303 spin_lock_irqsave(&mhu->doorbell_pending_lock, flags);
304
305 priv->pending = 1;
306 writel_relaxed(BIT(priv->doorbell),
307 &mhu->send->ch_wn[priv->ch_wn_idx].stat_set);
308
309 spin_unlock_irqrestore(lock: &mhu->doorbell_pending_lock, flags);
310
311 return 0;
312}
313
314static const struct mhuv2_protocol_ops mhuv2_doorbell_ops = {
315 .rx_startup = mhuv2_doorbell_rx_startup,
316 .rx_shutdown = mhuv2_doorbell_rx_shutdown,
317 .read_data = mhuv2_doorbell_read_data,
318 .last_tx_done = mhuv2_doorbell_last_tx_done,
319 .send_data = mhuv2_doorbell_send_data,
320};
321#define IS_PROTOCOL_DOORBELL(_priv) (_priv->ops == &mhuv2_doorbell_ops)
322
323/* ============= Data transfer transport protocol operations ================ */
324
325static int mhuv2_data_transfer_rx_startup(struct mhuv2 *mhu,
326 struct mbox_chan *chan)
327{
328 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
329 int i = priv->ch_wn_idx + priv->windows - 1;
330
331 /*
332 * The protocol mandates that all but the last status register must be
333 * masked.
334 */
335 writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_clear);
336 return 0;
337}
338
339static void mhuv2_data_transfer_rx_shutdown(struct mhuv2 *mhu,
340 struct mbox_chan *chan)
341{
342 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
343 int i = priv->ch_wn_idx + priv->windows - 1;
344
345 writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set);
346}
347
348static void *mhuv2_data_transfer_read_data(struct mhuv2 *mhu,
349 struct mbox_chan *chan)
350{
351 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
352 const int windows = priv->windows;
353 struct arm_mhuv2_mbox_msg *msg;
354 u32 *data;
355 int i, idx;
356
357 msg = kzalloc(size: sizeof(*msg) + windows * MHUV2_STAT_BYTES, GFP_KERNEL);
358 if (!msg)
359 return ERR_PTR(error: -ENOMEM);
360
361 data = msg->data = msg + 1;
362 msg->len = windows * MHUV2_STAT_BYTES;
363
364 /*
365 * Messages are expected in order of most significant word to least
366 * significant word. Refer mhuv2_data_transfer_send_data() for more
367 * details.
368 *
369 * We also need to read the stat register instead of stat_masked, as we
370 * masked all but the last window.
371 *
372 * Last channel window must be cleared as the final operation. Upon
373 * clearing the last channel window register, which is unmasked in
374 * data-transfer protocol, the interrupt is de-asserted.
375 */
376 for (i = 0; i < windows; i++) {
377 idx = priv->ch_wn_idx + i;
378 data[windows - 1 - i] = readl_relaxed(&mhu->recv->ch_wn[idx].stat);
379 writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[idx].stat_clear);
380 }
381
382 return msg;
383}
384
385static void mhuv2_data_transfer_tx_startup(struct mhuv2 *mhu,
386 struct mbox_chan *chan)
387{
388 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
389 int i = priv->ch_wn_idx + priv->windows - 1;
390
391 /* Enable interrupts only for the last window */
392 if (mhu->minor) {
393 writel_relaxed(0x1, &mhu->send->ch_wn[i].int_clr);
394 writel_relaxed(0x1, &mhu->send->ch_wn[i].int_en);
395 }
396}
397
398static void mhuv2_data_transfer_tx_shutdown(struct mhuv2 *mhu,
399 struct mbox_chan *chan)
400{
401 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
402 int i = priv->ch_wn_idx + priv->windows - 1;
403
404 if (mhu->minor)
405 writel_relaxed(0x0, &mhu->send->ch_wn[i].int_en);
406}
407
408static int mhuv2_data_transfer_last_tx_done(struct mhuv2 *mhu,
409 struct mbox_chan *chan)
410{
411 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
412 int i = priv->ch_wn_idx + priv->windows - 1;
413
414 /* Just checking the last channel window should be enough */
415 return !readl_relaxed(&mhu->send->ch_wn[i].stat);
416}
417
418/*
419 * Message will be transmitted from most significant to least significant word.
420 * This is to allow for messages shorter than channel windows to still trigger
421 * the receiver interrupt which gets activated when the last stat register is
422 * written. As an example, a 6-word message is to be written on a 4-channel MHU
423 * connection: Registers marked with '*' are masked, and will not generate an
424 * interrupt on the receiver side once written.
425 *
426 * u32 *data = [0x00000001], [0x00000002], [0x00000003], [0x00000004],
427 * [0x00000005], [0x00000006]
428 *
429 * ROUND 1:
430 * stat reg To write Write sequence
431 * [ stat 3 ] <- [0x00000001] 4 <- triggers interrupt on receiver
432 * [ stat 2 ] <- [0x00000002] 3
433 * [ stat 1 ] <- [0x00000003] 2
434 * [ stat 0 ] <- [0x00000004] 1
435 *
436 * data += 4 // Increment data pointer by number of stat regs
437 *
438 * ROUND 2:
439 * stat reg To write Write sequence
440 * [ stat 3 ] <- [0x00000005] 2 <- triggers interrupt on receiver
441 * [ stat 2 ] <- [0x00000006] 1
442 * [ stat 1 ] <- [0x00000000]
443 * [ stat 0 ] <- [0x00000000]
444 */
445static int mhuv2_data_transfer_send_data(struct mhuv2 *mhu,
446 struct mbox_chan *chan, void *arg)
447{
448 const struct arm_mhuv2_mbox_msg *msg = arg;
449 int bytes_left = msg->len, bytes_to_send, bytes_in_round, i;
450 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
451 int windows = priv->windows;
452 u32 *data = msg->data, word;
453
454 while (bytes_left) {
455 if (!data[0]) {
456 dev_err(mhu->mbox.dev, "Data aligned at first window can't be zero to guarantee interrupt generation at receiver");
457 return -EINVAL;
458 }
459
460 while(!mhuv2_data_transfer_last_tx_done(mhu, chan))
461 continue;
462
463 bytes_in_round = min(bytes_left, (int)(windows * MHUV2_STAT_BYTES));
464
465 for (i = windows - 1; i >= 0; i--) {
466 /* Data less than windows can transfer ? */
467 if (unlikely(bytes_in_round <= i * MHUV2_STAT_BYTES))
468 continue;
469
470 word = data[i];
471 bytes_to_send = bytes_in_round & (MHUV2_STAT_BYTES - 1);
472 if (unlikely(bytes_to_send))
473 word &= LSB_MASK(bytes_to_send);
474 else
475 bytes_to_send = MHUV2_STAT_BYTES;
476
477 writel_relaxed(word, &mhu->send->ch_wn[priv->ch_wn_idx + windows - 1 - i].stat_set);
478 bytes_left -= bytes_to_send;
479 bytes_in_round -= bytes_to_send;
480 }
481
482 data += windows;
483 }
484
485 return 0;
486}
487
488static const struct mhuv2_protocol_ops mhuv2_data_transfer_ops = {
489 .rx_startup = mhuv2_data_transfer_rx_startup,
490 .rx_shutdown = mhuv2_data_transfer_rx_shutdown,
491 .read_data = mhuv2_data_transfer_read_data,
492 .tx_startup = mhuv2_data_transfer_tx_startup,
493 .tx_shutdown = mhuv2_data_transfer_tx_shutdown,
494 .last_tx_done = mhuv2_data_transfer_last_tx_done,
495 .send_data = mhuv2_data_transfer_send_data,
496};
497
498/* Interrupt handlers */
499
500static struct mbox_chan *get_irq_chan_comb(struct mhuv2 *mhu, u32 __iomem *reg)
501{
502 struct mbox_chan *chans = mhu->mbox.chans;
503 int channel = 0, i, offset = 0, windows, protocol, ch_wn;
504 u32 stat;
505
506 for (i = 0; i < MHUV2_CMB_INT_ST_REG_CNT; i++) {
507 stat = readl_relaxed(reg + i);
508 if (!stat)
509 continue;
510
511 ch_wn = i * MHUV2_STAT_BITS + __builtin_ctz(stat);
512
513 for (i = 0; i < mhu->length; i += 2) {
514 protocol = mhu->protocols[i];
515 windows = mhu->protocols[i + 1];
516
517 if (ch_wn >= offset + windows) {
518 if (protocol == DOORBELL)
519 channel += MHUV2_STAT_BITS * windows;
520 else
521 channel++;
522
523 offset += windows;
524 continue;
525 }
526
527 /* Return first chan of the window in doorbell mode */
528 if (protocol == DOORBELL)
529 channel += MHUV2_STAT_BITS * (ch_wn - offset);
530
531 return &chans[channel];
532 }
533 }
534
535 return ERR_PTR(error: -EIO);
536}
537
538static irqreturn_t mhuv2_sender_interrupt(int irq, void *data)
539{
540 struct mhuv2 *mhu = data;
541 struct device *dev = mhu->mbox.dev;
542 struct mhuv2_mbox_chan_priv *priv;
543 struct mbox_chan *chan;
544 unsigned long flags;
545 int i, found = 0;
546 u32 stat;
547
548 chan = get_irq_chan_comb(mhu, reg: mhu->send->chcomb_int_st);
549 if (IS_ERR(ptr: chan)) {
550 dev_warn(dev, "Failed to find channel for the Tx interrupt\n");
551 return IRQ_NONE;
552 }
553 priv = chan->con_priv;
554
555 if (!IS_PROTOCOL_DOORBELL(priv)) {
556 for (i = 0; i < priv->windows; i++)
557 writel_relaxed(1, &mhu->send->ch_wn[priv->ch_wn_idx + i].int_clr);
558
559 if (chan->cl) {
560 mbox_chan_txdone(chan, r: 0);
561 return IRQ_HANDLED;
562 }
563
564 dev_warn(dev, "Tx interrupt Received on channel (%u) not currently attached to a mailbox client\n",
565 priv->ch_wn_idx);
566 return IRQ_NONE;
567 }
568
569 /* Clear the interrupt first, so we don't miss any doorbell later */
570 writel_relaxed(1, &mhu->send->ch_wn[priv->ch_wn_idx].int_clr);
571
572 /*
573 * In Doorbell mode, make sure no new transitions happen while the
574 * interrupt handler is trying to find the finished doorbell tx
575 * operations, else we may think few of the transfers were complete
576 * before they actually were.
577 */
578 spin_lock_irqsave(&mhu->doorbell_pending_lock, flags);
579
580 /*
581 * In case of doorbell mode, the first channel of the window is returned
582 * by get_irq_chan_comb(). Find all the pending channels here.
583 */
584 stat = readl_relaxed(&mhu->send->ch_wn[priv->ch_wn_idx].stat);
585
586 for (i = 0; i < MHUV2_STAT_BITS; i++) {
587 priv = chan[i].con_priv;
588
589 /* Find cases where pending was 1, but stat's bit is cleared */
590 if (priv->pending ^ ((stat >> i) & 0x1)) {
591 BUG_ON(!priv->pending);
592
593 if (!chan->cl) {
594 dev_warn(dev, "Tx interrupt received on doorbell (%u : %u) channel not currently attached to a mailbox client\n",
595 priv->ch_wn_idx, i);
596 continue;
597 }
598
599 mbox_chan_txdone(chan: &chan[i], r: 0);
600 priv->pending = 0;
601 found++;
602 }
603 }
604
605 spin_unlock_irqrestore(lock: &mhu->doorbell_pending_lock, flags);
606
607 if (!found) {
608 /*
609 * We may have already processed the doorbell in the previous
610 * iteration if the interrupt came right after we cleared it but
611 * before we read the stat register.
612 */
613 dev_dbg(dev, "Couldn't find the doorbell (%u) for the Tx interrupt interrupt\n",
614 priv->ch_wn_idx);
615 return IRQ_NONE;
616 }
617
618 return IRQ_HANDLED;
619}
620
621static struct mbox_chan *get_irq_chan_comb_rx(struct mhuv2 *mhu)
622{
623 struct mhuv2_mbox_chan_priv *priv;
624 struct mbox_chan *chan;
625 u32 stat;
626
627 chan = get_irq_chan_comb(mhu, reg: mhu->recv->chcomb_int_st);
628 if (IS_ERR(ptr: chan))
629 return chan;
630
631 priv = chan->con_priv;
632 if (!IS_PROTOCOL_DOORBELL(priv))
633 return chan;
634
635 /*
636 * In case of doorbell mode, the first channel of the window is returned
637 * by the routine. Find the exact channel here.
638 */
639 stat = readl_relaxed(&mhu->recv->ch_wn[priv->ch_wn_idx].stat_masked);
640 BUG_ON(!stat);
641
642 return chan + __builtin_ctz(stat);
643}
644
645static struct mbox_chan *get_irq_chan_stat_rx(struct mhuv2 *mhu)
646{
647 struct mbox_chan *chans = mhu->mbox.chans;
648 struct mhuv2_mbox_chan_priv *priv;
649 u32 stat;
650 int i = 0;
651
652 while (i < mhu->mbox.num_chans) {
653 priv = chans[i].con_priv;
654 stat = readl_relaxed(&mhu->recv->ch_wn[priv->ch_wn_idx].stat_masked);
655
656 if (stat) {
657 if (IS_PROTOCOL_DOORBELL(priv))
658 i += __builtin_ctz(stat);
659 return &chans[i];
660 }
661
662 i += IS_PROTOCOL_DOORBELL(priv) ? MHUV2_STAT_BITS : 1;
663 }
664
665 return ERR_PTR(error: -EIO);
666}
667
668static struct mbox_chan *get_irq_chan_rx(struct mhuv2 *mhu)
669{
670 if (!mhu->minor)
671 return get_irq_chan_stat_rx(mhu);
672
673 return get_irq_chan_comb_rx(mhu);
674}
675
676static irqreturn_t mhuv2_receiver_interrupt(int irq, void *arg)
677{
678 struct mhuv2 *mhu = arg;
679 struct mbox_chan *chan = get_irq_chan_rx(mhu);
680 struct device *dev = mhu->mbox.dev;
681 struct mhuv2_mbox_chan_priv *priv;
682 int ret = IRQ_NONE;
683 void *data;
684
685 if (IS_ERR(ptr: chan)) {
686 dev_warn(dev, "Failed to find channel for the rx interrupt\n");
687 return IRQ_NONE;
688 }
689 priv = chan->con_priv;
690
691 /* Read and clear the data first */
692 data = priv->ops->read_data(mhu, chan);
693
694 if (!chan->cl) {
695 dev_warn(dev, "Received data on channel (%u) not currently attached to a mailbox client\n",
696 priv->ch_wn_idx);
697 } else if (IS_ERR(ptr: data)) {
698 dev_err(dev, "Failed to read data: %lu\n", PTR_ERR(data));
699 } else {
700 mbox_chan_received_data(chan, data);
701 ret = IRQ_HANDLED;
702 }
703
704 if (!IS_ERR(ptr: data))
705 kfree(objp: data);
706
707 return ret;
708}
709
710/* Sender and receiver ops */
711static bool mhuv2_sender_last_tx_done(struct mbox_chan *chan)
712{
713 struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
714 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
715
716 return priv->ops->last_tx_done(mhu, chan);
717}
718
719static int mhuv2_sender_send_data(struct mbox_chan *chan, void *data)
720{
721 struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
722 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
723
724 if (!priv->ops->last_tx_done(mhu, chan))
725 return -EBUSY;
726
727 return priv->ops->send_data(mhu, chan, data);
728}
729
730static int mhuv2_sender_startup(struct mbox_chan *chan)
731{
732 struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
733 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
734
735 if (priv->ops->tx_startup)
736 priv->ops->tx_startup(mhu, chan);
737 return 0;
738}
739
740static void mhuv2_sender_shutdown(struct mbox_chan *chan)
741{
742 struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
743 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
744
745 if (priv->ops->tx_shutdown)
746 priv->ops->tx_shutdown(mhu, chan);
747}
748
749static const struct mbox_chan_ops mhuv2_sender_ops = {
750 .send_data = mhuv2_sender_send_data,
751 .startup = mhuv2_sender_startup,
752 .shutdown = mhuv2_sender_shutdown,
753 .last_tx_done = mhuv2_sender_last_tx_done,
754};
755
756static int mhuv2_receiver_startup(struct mbox_chan *chan)
757{
758 struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
759 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
760
761 return priv->ops->rx_startup(mhu, chan);
762}
763
764static void mhuv2_receiver_shutdown(struct mbox_chan *chan)
765{
766 struct mhuv2 *mhu = mhu_from_mbox(chan->mbox);
767 struct mhuv2_mbox_chan_priv *priv = chan->con_priv;
768
769 priv->ops->rx_shutdown(mhu, chan);
770}
771
772static int mhuv2_receiver_send_data(struct mbox_chan *chan, void *data)
773{
774 dev_err(chan->mbox->dev,
775 "Trying to transmit on a receiver MHU frame\n");
776 return -EIO;
777}
778
779static bool mhuv2_receiver_last_tx_done(struct mbox_chan *chan)
780{
781 dev_err(chan->mbox->dev, "Trying to Tx poll on a receiver MHU frame\n");
782 return true;
783}
784
785static const struct mbox_chan_ops mhuv2_receiver_ops = {
786 .send_data = mhuv2_receiver_send_data,
787 .startup = mhuv2_receiver_startup,
788 .shutdown = mhuv2_receiver_shutdown,
789 .last_tx_done = mhuv2_receiver_last_tx_done,
790};
791
792static struct mbox_chan *mhuv2_mbox_of_xlate(struct mbox_controller *mbox,
793 const struct of_phandle_args *pa)
794{
795 struct mhuv2 *mhu = mhu_from_mbox(mbox);
796 struct mbox_chan *chans = mbox->chans;
797 int channel = 0, i, offset, doorbell, protocol, windows;
798
799 if (pa->args_count != 2)
800 return ERR_PTR(error: -EINVAL);
801
802 offset = pa->args[0];
803 doorbell = pa->args[1];
804 if (doorbell >= MHUV2_STAT_BITS)
805 goto out;
806
807 for (i = 0; i < mhu->length; i += 2) {
808 protocol = mhu->protocols[i];
809 windows = mhu->protocols[i + 1];
810
811 if (protocol == DOORBELL) {
812 if (offset < windows)
813 return &chans[channel + MHUV2_STAT_BITS * offset + doorbell];
814
815 channel += MHUV2_STAT_BITS * windows;
816 offset -= windows;
817 } else {
818 if (offset == 0) {
819 if (doorbell)
820 goto out;
821
822 return &chans[channel];
823 }
824
825 channel++;
826 offset--;
827 }
828 }
829
830out:
831 dev_err(mbox->dev, "Couldn't xlate to a valid channel (%d: %d)\n",
832 pa->args[0], doorbell);
833 return ERR_PTR(error: -ENODEV);
834}
835
836static int mhuv2_verify_protocol(struct mhuv2 *mhu)
837{
838 struct device *dev = mhu->mbox.dev;
839 int protocol, windows, channels = 0, total_windows = 0, i;
840
841 for (i = 0; i < mhu->length; i += 2) {
842 protocol = mhu->protocols[i];
843 windows = mhu->protocols[i + 1];
844
845 if (!windows) {
846 dev_err(dev, "Window size can't be zero (%d)\n", i);
847 return -EINVAL;
848 }
849 total_windows += windows;
850
851 if (protocol == DOORBELL) {
852 channels += MHUV2_STAT_BITS * windows;
853 } else if (protocol == DATA_TRANSFER) {
854 channels++;
855 } else {
856 dev_err(dev, "Invalid protocol (%d) present in %s property at index %d\n",
857 protocol, MHUV2_PROTOCOL_PROP, i);
858 return -EINVAL;
859 }
860 }
861
862 if (total_windows > mhu->windows) {
863 dev_err(dev, "Channel windows can't be more than what's implemented by the hardware ( %d: %d)\n",
864 total_windows, mhu->windows);
865 return -EINVAL;
866 }
867
868 mhu->mbox.num_chans = channels;
869 return 0;
870}
871
872static int mhuv2_allocate_channels(struct mhuv2 *mhu)
873{
874 struct mbox_controller *mbox = &mhu->mbox;
875 struct mhuv2_mbox_chan_priv *priv;
876 struct device *dev = mbox->dev;
877 struct mbox_chan *chans;
878 int protocol, windows = 0, next_window = 0, i, j, k;
879
880 chans = devm_kcalloc(dev, n: mbox->num_chans, size: sizeof(*chans), GFP_KERNEL);
881 if (!chans)
882 return -ENOMEM;
883
884 mbox->chans = chans;
885
886 for (i = 0; i < mhu->length; i += 2) {
887 next_window += windows;
888
889 protocol = mhu->protocols[i];
890 windows = mhu->protocols[i + 1];
891
892 if (protocol == DATA_TRANSFER) {
893 priv = devm_kmalloc(dev, size: sizeof(*priv), GFP_KERNEL);
894 if (!priv)
895 return -ENOMEM;
896
897 priv->ch_wn_idx = next_window;
898 priv->ops = &mhuv2_data_transfer_ops;
899 priv->windows = windows;
900 chans++->con_priv = priv;
901 continue;
902 }
903
904 for (j = 0; j < windows; j++) {
905 for (k = 0; k < MHUV2_STAT_BITS; k++) {
906 priv = devm_kmalloc(dev, size: sizeof(*priv), GFP_KERNEL);
907 if (!priv)
908 return -ENOMEM;
909
910 priv->ch_wn_idx = next_window + j;
911 priv->ops = &mhuv2_doorbell_ops;
912 priv->doorbell = k;
913 chans++->con_priv = priv;
914 }
915
916 /*
917 * Permanently enable interrupt as we can't
918 * control it per doorbell.
919 */
920 if (mhu->frame == SENDER_FRAME && mhu->minor)
921 writel_relaxed(0x1, &mhu->send->ch_wn[priv->ch_wn_idx].int_en);
922 }
923 }
924
925 /* Make sure we have initialized all channels */
926 BUG_ON(chans - mbox->chans != mbox->num_chans);
927
928 return 0;
929}
930
931static int mhuv2_parse_channels(struct mhuv2 *mhu)
932{
933 struct device *dev = mhu->mbox.dev;
934 const struct device_node *np = dev->of_node;
935 int ret, count;
936 u32 *protocols;
937
938 count = of_property_count_u32_elems(np, MHUV2_PROTOCOL_PROP);
939 if (count <= 0 || count % 2) {
940 dev_err(dev, "Invalid %s property (%d)\n", MHUV2_PROTOCOL_PROP,
941 count);
942 return -EINVAL;
943 }
944
945 protocols = devm_kmalloc_array(dev, n: count, size: sizeof(*protocols), GFP_KERNEL);
946 if (!protocols)
947 return -ENOMEM;
948
949 ret = of_property_read_u32_array(np, MHUV2_PROTOCOL_PROP, out_values: protocols, sz: count);
950 if (ret) {
951 dev_err(dev, "Failed to read %s property: %d\n",
952 MHUV2_PROTOCOL_PROP, ret);
953 return ret;
954 }
955
956 mhu->protocols = protocols;
957 mhu->length = count;
958
959 ret = mhuv2_verify_protocol(mhu);
960 if (ret)
961 return ret;
962
963 return mhuv2_allocate_channels(mhu);
964}
965
966static int mhuv2_tx_init(struct amba_device *adev, struct mhuv2 *mhu,
967 void __iomem *reg)
968{
969 struct device *dev = mhu->mbox.dev;
970 int ret, i;
971
972 mhu->frame = SENDER_FRAME;
973 mhu->mbox.ops = &mhuv2_sender_ops;
974 mhu->send = reg;
975
976 mhu->windows = readl_relaxed_bitfield(&mhu->send->mhu_cfg, struct mhu_cfg_t, num_ch);
977 mhu->minor = readl_relaxed_bitfield(&mhu->send->aidr, struct aidr_t, arch_minor_rev);
978
979 spin_lock_init(&mhu->doorbell_pending_lock);
980
981 /*
982 * For minor version 1 and forward, tx interrupt is provided by
983 * the controller.
984 */
985 if (mhu->minor && adev->irq[0]) {
986 ret = devm_request_threaded_irq(dev, irq: adev->irq[0], NULL,
987 thread_fn: mhuv2_sender_interrupt,
988 IRQF_ONESHOT, devname: "mhuv2-tx", dev_id: mhu);
989 if (ret) {
990 dev_err(dev, "Failed to request tx IRQ, fallback to polling mode: %d\n",
991 ret);
992 } else {
993 mhu->mbox.txdone_irq = true;
994 mhu->mbox.txdone_poll = false;
995 mhu->irq = adev->irq[0];
996
997 writel_relaxed_bitfield(1, &mhu->send->int_en, struct int_en_t, chcomb);
998
999 /* Disable all channel interrupts */
1000 for (i = 0; i < mhu->windows; i++)
1001 writel_relaxed(0x0, &mhu->send->ch_wn[i].int_en);
1002
1003 goto out;
1004 }
1005 }
1006
1007 mhu->mbox.txdone_irq = false;
1008 mhu->mbox.txdone_poll = true;
1009 mhu->mbox.txpoll_period = 1;
1010
1011out:
1012 /* Wait for receiver to be ready */
1013 writel_relaxed(0x1, &mhu->send->access_request);
1014 while (!readl_relaxed(&mhu->send->access_ready))
1015 continue;
1016
1017 return 0;
1018}
1019
1020static int mhuv2_rx_init(struct amba_device *adev, struct mhuv2 *mhu,
1021 void __iomem *reg)
1022{
1023 struct device *dev = mhu->mbox.dev;
1024 int ret, i;
1025
1026 mhu->frame = RECEIVER_FRAME;
1027 mhu->mbox.ops = &mhuv2_receiver_ops;
1028 mhu->recv = reg;
1029
1030 mhu->windows = readl_relaxed_bitfield(&mhu->recv->mhu_cfg, struct mhu_cfg_t, num_ch);
1031 mhu->minor = readl_relaxed_bitfield(&mhu->recv->aidr, struct aidr_t, arch_minor_rev);
1032
1033 mhu->irq = adev->irq[0];
1034 if (!mhu->irq) {
1035 dev_err(dev, "Missing receiver IRQ\n");
1036 return -EINVAL;
1037 }
1038
1039 ret = devm_request_threaded_irq(dev, irq: mhu->irq, NULL,
1040 thread_fn: mhuv2_receiver_interrupt, IRQF_ONESHOT,
1041 devname: "mhuv2-rx", dev_id: mhu);
1042 if (ret) {
1043 dev_err(dev, "Failed to request rx IRQ\n");
1044 return ret;
1045 }
1046
1047 /* Mask all the channel windows */
1048 for (i = 0; i < mhu->windows; i++)
1049 writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set);
1050
1051 if (mhu->minor)
1052 writel_relaxed_bitfield(1, &mhu->recv->int_en, struct int_en_t, chcomb);
1053
1054 return 0;
1055}
1056
1057static int mhuv2_probe(struct amba_device *adev, const struct amba_id *id)
1058{
1059 struct device *dev = &adev->dev;
1060 const struct device_node *np = dev->of_node;
1061 struct mhuv2 *mhu;
1062 void __iomem *reg;
1063 int ret = -EINVAL;
1064
1065 reg = devm_of_iomap(dev, node: dev->of_node, index: 0, NULL);
1066 if (IS_ERR(ptr: reg))
1067 return PTR_ERR(ptr: reg);
1068
1069 mhu = devm_kzalloc(dev, size: sizeof(*mhu), GFP_KERNEL);
1070 if (!mhu)
1071 return -ENOMEM;
1072
1073 mhu->mbox.dev = dev;
1074 mhu->mbox.of_xlate = mhuv2_mbox_of_xlate;
1075
1076 if (of_device_is_compatible(device: np, "arm,mhuv2-tx"))
1077 ret = mhuv2_tx_init(adev, mhu, reg);
1078 else if (of_device_is_compatible(device: np, "arm,mhuv2-rx"))
1079 ret = mhuv2_rx_init(adev, mhu, reg);
1080 else
1081 dev_err(dev, "Invalid compatible property\n");
1082
1083 if (ret)
1084 return ret;
1085
1086 /* Channel windows can't be 0 */
1087 BUG_ON(!mhu->windows);
1088
1089 ret = mhuv2_parse_channels(mhu);
1090 if (ret)
1091 return ret;
1092
1093 amba_set_drvdata(adev, mhu);
1094
1095 ret = devm_mbox_controller_register(dev, mbox: &mhu->mbox);
1096 if (ret)
1097 dev_err(dev, "failed to register ARM MHUv2 driver %d\n", ret);
1098
1099 return ret;
1100}
1101
1102static void mhuv2_remove(struct amba_device *adev)
1103{
1104 struct mhuv2 *mhu = amba_get_drvdata(adev);
1105
1106 if (mhu->frame == SENDER_FRAME)
1107 writel_relaxed(0x0, &mhu->send->access_request);
1108}
1109
1110static struct amba_id mhuv2_ids[] = {
1111 {
1112 /* 2.0 */
1113 .id = 0xbb0d1,
1114 .mask = 0xfffff,
1115 },
1116 {
1117 /* 2.1 */
1118 .id = 0xbb076,
1119 .mask = 0xfffff,
1120 },
1121 { 0, 0 },
1122};
1123MODULE_DEVICE_TABLE(amba, mhuv2_ids);
1124
1125static struct amba_driver mhuv2_driver = {
1126 .drv = {
1127 .name = "arm-mhuv2",
1128 },
1129 .id_table = mhuv2_ids,
1130 .probe = mhuv2_probe,
1131 .remove = mhuv2_remove,
1132};
1133module_amba_driver(mhuv2_driver);
1134
1135MODULE_LICENSE("GPL v2");
1136MODULE_DESCRIPTION("ARM MHUv2 Driver");
1137MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>");
1138MODULE_AUTHOR("Tushar Khandelwal <tushar.khandelwal@arm.com>");
1139

source code of linux/drivers/mailbox/arm_mhuv2.c