1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ARM Message Handling Unit Version 3 (MHUv3) driver.
4 *
5 * Copyright (C) 2024 ARM Ltd.
6 *
7 * Based on ARM MHUv2 driver.
8 */
9
10#include <linux/bitfield.h>
11#include <linux/bitops.h>
12#include <linux/bits.h>
13#include <linux/cleanup.h>
14#include <linux/device.h>
15#include <linux/interrupt.h>
16#include <linux/mailbox_controller.h>
17#include <linux/module.h>
18#include <linux/of_address.h>
19#include <linux/platform_device.h>
20#include <linux/spinlock.h>
21#include <linux/sizes.h>
22#include <linux/slab.h>
23#include <linux/types.h>
24
25/* ====== MHUv3 Registers ====== */
26
27/* Maximum number of Doorbell channel windows */
28#define MHUV3_DBCW_MAX 128
29/* Number of DBCH combined interrupt status registers */
30#define MHUV3_DBCH_CMB_INT_ST_REG_CNT 4
31
32/* Number of FFCH combined interrupt status registers */
33#define MHUV3_FFCH_CMB_INT_ST_REG_CNT 2
34
35#define MHUV3_FLAG_BITS 32
36
37/* Not a typo ... */
38#define MHUV3_MAJOR_VERSION 2
39
40enum {
41 MHUV3_MBOX_CELL_TYPE,
42 MHUV3_MBOX_CELL_CHWN,
43 MHUV3_MBOX_CELL_PARAM,
44 MHUV3_MBOX_CELLS
45};
46
47/* Padding bitfields/fields represents hole in the regs MMIO */
48
49/* CTRL_Page */
50struct blk_id {
51#define id GENMASK(3, 0)
52 u32 val;
53} __packed;
54
55struct feat_spt0 {
56#define dbe_spt GENMASK(3, 0)
57#define fe_spt GENMASK(7, 4)
58#define fce_spt GENMASK(11, 8)
59 u32 val;
60} __packed;
61
62struct feat_spt1 {
63#define auto_op_spt GENMASK(3, 0)
64 u32 val;
65} __packed;
66
67struct dbch_cfg0 {
68#define num_dbch GENMASK(7, 0)
69 u32 val;
70} __packed;
71
72struct ffch_cfg0 {
73#define num_ffch GENMASK(7, 0)
74#define x8ba_spt BIT(8)
75#define x16ba_spt BIT(9)
76#define x32ba_spt BIT(10)
77#define x64ba_spt BIT(11)
78#define ffch_depth GENMASK(25, 16)
79 u32 val;
80} __packed;
81
82struct fch_cfg0 {
83#define num_fch GENMASK(9, 0)
84#define fcgi_spt BIT(10) // MBX-only
85#define num_fcg GENMASK(15, 11)
86#define num_fch_per_grp GENMASK(20, 16)
87#define fch_ws GENMASK(28, 21)
88 u32 val;
89} __packed;
90
91struct ctrl {
92#define op_req BIT(0)
93#define ch_op_mask BIT(1)
94 u32 val;
95} __packed;
96
97struct fch_ctrl {
98#define _int_en BIT(2)
99 u32 val;
100} __packed;
101
102struct iidr {
103#define implementer GENMASK(11, 0)
104#define revision GENMASK(15, 12)
105#define variant GENMASK(19, 16)
106#define product_id GENMASK(31, 20)
107 u32 val;
108} __packed;
109
110struct aidr {
111#define arch_minor_rev GENMASK(3, 0)
112#define arch_major_rev GENMASK(7, 4)
113 u32 val;
114} __packed;
115
116struct ctrl_page {
117 struct blk_id blk_id;
118 u8 pad[12];
119 struct feat_spt0 feat_spt0;
120 struct feat_spt1 feat_spt1;
121 u8 pad1[8];
122 struct dbch_cfg0 dbch_cfg0;
123 u8 pad2[12];
124 struct ffch_cfg0 ffch_cfg0;
125 u8 pad3[12];
126 struct fch_cfg0 fch_cfg0;
127 u8 pad4[188];
128 struct ctrl x_ctrl;
129 /*-- MBX-only registers --*/
130 u8 pad5[60];
131 struct fch_ctrl fch_ctrl;
132 u32 fcg_int_en;
133 u8 pad6[696];
134 /*-- End of MBX-only ---- */
135 u32 dbch_int_st[MHUV3_DBCH_CMB_INT_ST_REG_CNT];
136 u32 ffch_int_st[MHUV3_FFCH_CMB_INT_ST_REG_CNT];
137 /*-- MBX-only registers --*/
138 u8 pad7[88];
139 u32 fcg_int_st;
140 u8 pad8[12];
141 u32 fcg_grp_int_st[32];
142 u8 pad9[2760];
143 /*-- End of MBX-only ---- */
144 struct iidr iidr;
145 struct aidr aidr;
146 u32 imp_def_id[12];
147} __packed;
148
149/* DBCW_Page */
150
151struct xbcw_ctrl {
152#define comb_en BIT(0)
153 u32 val;
154} __packed;
155
156struct pdbcw_int {
157#define tfr_ack BIT(0)
158 u32 val;
159} __packed;
160
161struct pdbcw_page {
162 u32 st;
163 u8 pad[8];
164 u32 set;
165 struct pdbcw_int int_st;
166 struct pdbcw_int int_clr;
167 struct pdbcw_int int_en;
168 struct xbcw_ctrl ctrl;
169} __packed;
170
171struct mdbcw_page {
172 u32 st;
173 u32 st_msk;
174 u32 clr;
175 u8 pad[4];
176 u32 msk_st;
177 u32 msk_set;
178 u32 msk_clr;
179 struct xbcw_ctrl ctrl;
180} __packed;
181
182struct dummy_page {
183 u8 pad[SZ_4K];
184} __packed;
185
186struct mhu3_pbx_frame_reg {
187 struct ctrl_page ctrl;
188 struct pdbcw_page dbcw[MHUV3_DBCW_MAX];
189 struct dummy_page ffcw;
190 struct dummy_page fcw;
191 u8 pad[SZ_4K * 11];
192 struct dummy_page impdef;
193} __packed;
194
195struct mhu3_mbx_frame_reg {
196 struct ctrl_page ctrl;
197 struct mdbcw_page dbcw[MHUV3_DBCW_MAX];
198 struct dummy_page ffcw;
199 struct dummy_page fcw;
200 u8 pad[SZ_4K * 11];
201 struct dummy_page impdef;
202} __packed;
203
204/* Macro for reading a bitmask within a physically mapped packed struct */
205#define readl_relaxed_bitmask(_regptr, _bitmask) \
206 ({ \
207 unsigned long _rval; \
208 _rval = readl_relaxed(_regptr); \
209 FIELD_GET(_bitmask, _rval); \
210 })
211
212/* Macro for writing a bitmask within a physically mapped packed struct */
213#define writel_relaxed_bitmask(_value, _regptr, _bitmask) \
214 ({ \
215 unsigned long _rval; \
216 typeof(_regptr) _rptr = _regptr; \
217 typeof(_bitmask) _bmask = _bitmask; \
218 _rval = readl_relaxed(_rptr); \
219 _rval &= ~(_bmask); \
220 _rval |= FIELD_PREP((unsigned long long)_bmask, _value);\
221 writel_relaxed(_rval, _rptr); \
222 })
223
224/* ====== MHUv3 data structures ====== */
225
226enum mhuv3_frame {
227 PBX_FRAME,
228 MBX_FRAME,
229};
230
231static char *mhuv3_str[] = {
232 "PBX",
233 "MBX"
234};
235
236enum mhuv3_extension_type {
237 DBE_EXT,
238 FCE_EXT,
239 FE_EXT,
240 NUM_EXT
241};
242
243static char *mhuv3_ext_str[] = {
244 "DBE",
245 "FCE",
246 "FE"
247};
248
249struct mhuv3;
250
251/**
252 * struct mhuv3_protocol_ops - MHUv3 operations
253 *
254 * @rx_startup: Receiver startup callback.
255 * @rx_shutdown: Receiver shutdown callback.
256 * @read_data: Read available Sender in-band LE data (if any).
257 * @rx_complete: Acknowledge data reception to the Sender. Any out-of-band data
258 * has to have been already retrieved before calling this.
259 * @tx_startup: Sender startup callback.
260 * @tx_shutdown: Sender shutdown callback.
261 * @last_tx_done: Report back to the Sender if the last transfer has completed.
262 * @send_data: Send data to the receiver.
263 *
264 * Each supported transport protocol provides its own implementation of
265 * these operations.
266 */
267struct mhuv3_protocol_ops {
268 int (*rx_startup)(struct mhuv3 *mhu, struct mbox_chan *chan);
269 void (*rx_shutdown)(struct mhuv3 *mhu, struct mbox_chan *chan);
270 void *(*read_data)(struct mhuv3 *mhu, struct mbox_chan *chan);
271 void (*rx_complete)(struct mhuv3 *mhu, struct mbox_chan *chan);
272 void (*tx_startup)(struct mhuv3 *mhu, struct mbox_chan *chan);
273 void (*tx_shutdown)(struct mhuv3 *mhu, struct mbox_chan *chan);
274 int (*last_tx_done)(struct mhuv3 *mhu, struct mbox_chan *chan);
275 int (*send_data)(struct mhuv3 *mhu, struct mbox_chan *chan, void *arg);
276};
277
278/**
279 * struct mhuv3_mbox_chan_priv - MHUv3 channel private information
280 *
281 * @ch_idx: Channel window index associated to this mailbox channel.
282 * @doorbell: Doorbell bit number within the @ch_idx window.
283 * Only relevant to Doorbell transport.
284 * @ops: Transport protocol specific operations for this channel.
285 *
286 * Transport specific data attached to mmailbox channel priv data.
287 */
288struct mhuv3_mbox_chan_priv {
289 u32 ch_idx;
290 u32 doorbell;
291 const struct mhuv3_protocol_ops *ops;
292};
293
294/**
295 * struct mhuv3_extension - MHUv3 extension descriptor
296 *
297 * @type: Type of extension
298 * @num_chans: Max number of channels found for this extension.
299 * @base_ch_idx: First channel number assigned to this extension, picked from
300 * the set of all mailbox channels descriptors created.
301 * @mbox_of_xlate: Extension specific helper to parse DT and lookup associated
302 * channel from the related 'mboxes' property.
303 * @combined_irq_setup: Extension specific helper to setup the combined irq.
304 * @channels_init: Extension specific helper to initialize channels.
305 * @chan_from_comb_irq_get: Extension specific helper to lookup which channel
306 * triggered the combined irq.
307 * @pending_db: Array of per-channel pending doorbells.
308 * @pending_lock: Protect access to pending_db.
309 */
310struct mhuv3_extension {
311 enum mhuv3_extension_type type;
312 unsigned int num_chans;
313 unsigned int base_ch_idx;
314 struct mbox_chan *(*mbox_of_xlate)(struct mhuv3 *mhu,
315 unsigned int channel,
316 unsigned int param);
317 void (*combined_irq_setup)(struct mhuv3 *mhu);
318 int (*channels_init)(struct mhuv3 *mhu);
319 struct mbox_chan *(*chan_from_comb_irq_get)(struct mhuv3 *mhu);
320 u32 pending_db[MHUV3_DBCW_MAX];
321 /* Protect access to pending_db */
322 spinlock_t pending_lock;
323};
324
325/**
326 * struct mhuv3 - MHUv3 mailbox controller data
327 *
328 * @frame: Frame type: MBX_FRAME or PBX_FRAME.
329 * @auto_op_full: Flag to indicate if the MHU supports AutoOp full mode.
330 * @major: MHUv3 controller architectural major version.
331 * @minor: MHUv3 controller architectural minor version.
332 * @implem: MHUv3 controller IIDR implementer.
333 * @rev: MHUv3 controller IIDR revision.
334 * @var: MHUv3 controller IIDR variant.
335 * @prod_id: MHUv3 controller IIDR product_id.
336 * @num_chans: The total number of channnels discovered across all extensions.
337 * @cmb_irq: Combined IRQ number if any found defined.
338 * @ctrl: A reference to the MHUv3 control page for this block.
339 * @pbx: Base address of the PBX register mapping region.
340 * @mbx: Base address of the MBX register mapping region.
341 * @ext: Array holding descriptors for any found implemented extension.
342 * @mbox: Mailbox controller belonging to the MHU frame.
343 */
344struct mhuv3 {
345 enum mhuv3_frame frame;
346 bool auto_op_full;
347 unsigned int major;
348 unsigned int minor;
349 unsigned int implem;
350 unsigned int rev;
351 unsigned int var;
352 unsigned int prod_id;
353 unsigned int num_chans;
354 int cmb_irq;
355 struct ctrl_page __iomem *ctrl;
356 union {
357 struct mhu3_pbx_frame_reg __iomem *pbx;
358 struct mhu3_mbx_frame_reg __iomem *mbx;
359 };
360 struct mhuv3_extension *ext[NUM_EXT];
361 struct mbox_controller mbox;
362};
363
364#define mhu_from_mbox(_mbox) container_of(_mbox, struct mhuv3, mbox)
365
366typedef int (*mhuv3_extension_initializer)(struct mhuv3 *mhu);
367
368/* =================== Doorbell transport protocol operations =============== */
369
370static void mhuv3_doorbell_tx_startup(struct mhuv3 *mhu, struct mbox_chan *chan)
371{
372 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
373
374 /* Enable Transfer Acknowledgment events */
375 writel_relaxed_bitmask(0x1, &mhu->pbx->dbcw[priv->ch_idx].int_en, tfr_ack);
376}
377
378static void mhuv3_doorbell_tx_shutdown(struct mhuv3 *mhu, struct mbox_chan *chan)
379{
380 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
381 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
382 unsigned long flags;
383
384 /* Disable Channel Transfer Ack events */
385 writel_relaxed_bitmask(0x0, &mhu->pbx->dbcw[priv->ch_idx].int_en, tfr_ack);
386
387 /* Clear Channel Transfer Ack and pending doorbells */
388 writel_relaxed_bitmask(0x1, &mhu->pbx->dbcw[priv->ch_idx].int_clr, tfr_ack);
389 spin_lock_irqsave(&e->pending_lock, flags);
390 e->pending_db[priv->ch_idx] = 0;
391 spin_unlock_irqrestore(lock: &e->pending_lock, flags);
392}
393
394static int mhuv3_doorbell_rx_startup(struct mhuv3 *mhu, struct mbox_chan *chan)
395{
396 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
397
398 /* Unmask Channel Transfer events */
399 writel_relaxed(BIT(priv->doorbell), &mhu->mbx->dbcw[priv->ch_idx].msk_clr);
400
401 return 0;
402}
403
404static void mhuv3_doorbell_rx_shutdown(struct mhuv3 *mhu,
405 struct mbox_chan *chan)
406{
407 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
408
409 /* Mask Channel Transfer events */
410 writel_relaxed(BIT(priv->doorbell), &mhu->mbx->dbcw[priv->ch_idx].msk_set);
411}
412
413static void mhuv3_doorbell_rx_complete(struct mhuv3 *mhu, struct mbox_chan *chan)
414{
415 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
416
417 /* Clearing the pending transfer generates the Channel Transfer Ack */
418 writel_relaxed(BIT(priv->doorbell), &mhu->mbx->dbcw[priv->ch_idx].clr);
419}
420
421static int mhuv3_doorbell_last_tx_done(struct mhuv3 *mhu,
422 struct mbox_chan *chan)
423{
424 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
425 int done;
426
427 done = !(readl_relaxed(&mhu->pbx->dbcw[priv->ch_idx].st) &
428 BIT(priv->doorbell));
429 if (done) {
430 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
431 unsigned long flags;
432
433 /* Take care to clear the pending doorbell also when polling */
434 spin_lock_irqsave(&e->pending_lock, flags);
435 e->pending_db[priv->ch_idx] &= ~BIT(priv->doorbell);
436 spin_unlock_irqrestore(lock: &e->pending_lock, flags);
437 }
438
439 return done;
440}
441
442static int mhuv3_doorbell_send_data(struct mhuv3 *mhu, struct mbox_chan *chan,
443 void *arg)
444{
445 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
446 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
447
448 scoped_guard(spinlock_irqsave, &e->pending_lock) {
449 /* Only one in-flight Transfer is allowed per-doorbell */
450 if (e->pending_db[priv->ch_idx] & BIT(priv->doorbell))
451 return -EBUSY;
452
453 e->pending_db[priv->ch_idx] |= BIT(priv->doorbell);
454 }
455
456 writel_relaxed(BIT(priv->doorbell), &mhu->pbx->dbcw[priv->ch_idx].set);
457
458 return 0;
459}
460
461static const struct mhuv3_protocol_ops mhuv3_doorbell_ops = {
462 .tx_startup = mhuv3_doorbell_tx_startup,
463 .tx_shutdown = mhuv3_doorbell_tx_shutdown,
464 .rx_startup = mhuv3_doorbell_rx_startup,
465 .rx_shutdown = mhuv3_doorbell_rx_shutdown,
466 .rx_complete = mhuv3_doorbell_rx_complete,
467 .last_tx_done = mhuv3_doorbell_last_tx_done,
468 .send_data = mhuv3_doorbell_send_data,
469};
470
471/* Sender and receiver mailbox ops */
472static bool mhuv3_sender_last_tx_done(struct mbox_chan *chan)
473{
474 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
475 struct mhuv3 *mhu = mhu_from_mbox(chan->mbox);
476
477 return priv->ops->last_tx_done(mhu, chan);
478}
479
480static int mhuv3_sender_send_data(struct mbox_chan *chan, void *data)
481{
482 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
483 struct mhuv3 *mhu = mhu_from_mbox(chan->mbox);
484
485 if (!priv->ops->last_tx_done(mhu, chan))
486 return -EBUSY;
487
488 return priv->ops->send_data(mhu, chan, data);
489}
490
491static int mhuv3_sender_startup(struct mbox_chan *chan)
492{
493 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
494 struct mhuv3 *mhu = mhu_from_mbox(chan->mbox);
495
496 if (priv->ops->tx_startup)
497 priv->ops->tx_startup(mhu, chan);
498
499 return 0;
500}
501
502static void mhuv3_sender_shutdown(struct mbox_chan *chan)
503{
504 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
505 struct mhuv3 *mhu = mhu_from_mbox(chan->mbox);
506
507 if (priv->ops->tx_shutdown)
508 priv->ops->tx_shutdown(mhu, chan);
509}
510
511static const struct mbox_chan_ops mhuv3_sender_ops = {
512 .send_data = mhuv3_sender_send_data,
513 .startup = mhuv3_sender_startup,
514 .shutdown = mhuv3_sender_shutdown,
515 .last_tx_done = mhuv3_sender_last_tx_done,
516};
517
518static int mhuv3_receiver_startup(struct mbox_chan *chan)
519{
520 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
521 struct mhuv3 *mhu = mhu_from_mbox(chan->mbox);
522
523 return priv->ops->rx_startup(mhu, chan);
524}
525
526static void mhuv3_receiver_shutdown(struct mbox_chan *chan)
527{
528 struct mhuv3_mbox_chan_priv *priv = chan->con_priv;
529 struct mhuv3 *mhu = mhu_from_mbox(chan->mbox);
530
531 priv->ops->rx_shutdown(mhu, chan);
532}
533
534static int mhuv3_receiver_send_data(struct mbox_chan *chan, void *data)
535{
536 dev_err(chan->mbox->dev,
537 "Trying to transmit on a MBX MHUv3 frame\n");
538 return -EIO;
539}
540
541static bool mhuv3_receiver_last_tx_done(struct mbox_chan *chan)
542{
543 dev_err(chan->mbox->dev, "Trying to Tx poll on a MBX MHUv3 frame\n");
544 return true;
545}
546
547static const struct mbox_chan_ops mhuv3_receiver_ops = {
548 .send_data = mhuv3_receiver_send_data,
549 .startup = mhuv3_receiver_startup,
550 .shutdown = mhuv3_receiver_shutdown,
551 .last_tx_done = mhuv3_receiver_last_tx_done,
552};
553
554static struct mbox_chan *mhuv3_dbe_mbox_of_xlate(struct mhuv3 *mhu,
555 unsigned int channel,
556 unsigned int doorbell)
557{
558 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
559 struct mbox_controller *mbox = &mhu->mbox;
560 struct mbox_chan *chans = mbox->chans;
561
562 if (channel >= e->num_chans || doorbell >= MHUV3_FLAG_BITS) {
563 dev_err(mbox->dev, "Couldn't xlate to a valid channel (%d: %d)\n",
564 channel, doorbell);
565 return ERR_PTR(error: -ENODEV);
566 }
567
568 return &chans[e->base_ch_idx + channel * MHUV3_FLAG_BITS + doorbell];
569}
570
571static void mhuv3_dbe_combined_irq_setup(struct mhuv3 *mhu)
572{
573 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
574 int i;
575
576 if (mhu->frame == PBX_FRAME) {
577 struct pdbcw_page __iomem *dbcw = mhu->pbx->dbcw;
578
579 for (i = 0; i < e->num_chans; i++) {
580 writel_relaxed_bitmask(0x1, &dbcw[i].int_clr, tfr_ack);
581 writel_relaxed_bitmask(0x0, &dbcw[i].int_en, tfr_ack);
582 writel_relaxed_bitmask(0x1, &dbcw[i].ctrl, comb_en);
583 }
584 } else {
585 struct mdbcw_page __iomem *dbcw = mhu->mbx->dbcw;
586
587 for (i = 0; i < e->num_chans; i++) {
588 writel_relaxed(0xFFFFFFFF, &dbcw[i].clr);
589 writel_relaxed(0xFFFFFFFF, &dbcw[i].msk_set);
590 writel_relaxed_bitmask(0x1, &dbcw[i].ctrl, comb_en);
591 }
592 }
593}
594
595static int mhuv3_dbe_channels_init(struct mhuv3 *mhu)
596{
597 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
598 struct mbox_controller *mbox = &mhu->mbox;
599 struct mbox_chan *chans;
600 int i;
601
602 chans = mbox->chans + mbox->num_chans;
603 e->base_ch_idx = mbox->num_chans;
604 for (i = 0; i < e->num_chans; i++) {
605 struct mhuv3_mbox_chan_priv *priv;
606 int k;
607
608 for (k = 0; k < MHUV3_FLAG_BITS; k++) {
609 priv = devm_kmalloc(dev: mbox->dev, size: sizeof(*priv), GFP_KERNEL);
610 if (!priv)
611 return -ENOMEM;
612
613 priv->ch_idx = i;
614 priv->ops = &mhuv3_doorbell_ops;
615 priv->doorbell = k;
616 chans++->con_priv = priv;
617 mbox->num_chans++;
618 }
619 }
620
621 spin_lock_init(&e->pending_lock);
622
623 return 0;
624}
625
626static bool mhuv3_dbe_doorbell_lookup(struct mhuv3 *mhu, unsigned int channel,
627 unsigned int *db)
628{
629 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
630 struct device *dev = mhu->mbox.dev;
631 u32 st;
632
633 if (mhu->frame == PBX_FRAME) {
634 u32 active_dbs, fired_dbs;
635
636 st = readl_relaxed_bitmask(&mhu->pbx->dbcw[channel].int_st,
637 tfr_ack);
638 if (!st)
639 goto err_spurious;
640
641 active_dbs = readl_relaxed(&mhu->pbx->dbcw[channel].st);
642 scoped_guard(spinlock_irqsave, &e->pending_lock) {
643 fired_dbs = e->pending_db[channel] & ~active_dbs;
644 if (!fired_dbs)
645 goto err_spurious;
646
647 *db = __ffs(fired_dbs);
648 e->pending_db[channel] &= ~BIT(*db);
649 }
650 fired_dbs &= ~BIT(*db);
651 /* Clear TFR Ack if no more doorbells pending */
652 if (!fired_dbs)
653 writel_relaxed_bitmask(0x1,
654 &mhu->pbx->dbcw[channel].int_clr,
655 tfr_ack);
656 } else {
657 st = readl_relaxed(&mhu->mbx->dbcw[channel].st_msk);
658 if (!st)
659 goto err_spurious;
660
661 *db = __ffs(st);
662 }
663
664 return true;
665
666err_spurious:
667 dev_warn(dev, "Spurious IRQ on %s channel:%d\n",
668 mhuv3_str[mhu->frame], channel);
669
670 return false;
671}
672
673static struct mbox_chan *mhuv3_dbe_chan_from_comb_irq_get(struct mhuv3 *mhu)
674{
675 struct mhuv3_extension *e = mhu->ext[DBE_EXT];
676 struct device *dev = mhu->mbox.dev;
677 int i;
678
679 for (i = 0; i < MHUV3_DBCH_CMB_INT_ST_REG_CNT; i++) {
680 unsigned int channel, db;
681 u32 cmb_st;
682
683 cmb_st = readl_relaxed(&mhu->ctrl->dbch_int_st[i]);
684 if (!cmb_st)
685 continue;
686
687 channel = i * MHUV3_FLAG_BITS + __ffs(cmb_st);
688 if (channel >= e->num_chans) {
689 dev_err(dev, "Invalid %s channel:%d\n",
690 mhuv3_str[mhu->frame], channel);
691 return ERR_PTR(error: -EIO);
692 }
693
694 if (!mhuv3_dbe_doorbell_lookup(mhu, channel, db: &db))
695 continue;
696
697 dev_dbg(dev, "Found %s ch[%d]/db[%d]\n",
698 mhuv3_str[mhu->frame], channel, db);
699
700 return &mhu->mbox.chans[channel * MHUV3_FLAG_BITS + db];
701 }
702
703 return ERR_PTR(error: -EIO);
704}
705
706static int mhuv3_dbe_init(struct mhuv3 *mhu)
707{
708 struct device *dev = mhu->mbox.dev;
709 struct mhuv3_extension *e;
710
711 if (!readl_relaxed_bitmask(&mhu->ctrl->feat_spt0, dbe_spt))
712 return 0;
713
714 dev_dbg(dev, "%s: Initializing DBE Extension.\n", mhuv3_str[mhu->frame]);
715
716 e = devm_kzalloc(dev, size: sizeof(*e), GFP_KERNEL);
717 if (!e)
718 return -ENOMEM;
719
720 e->type = DBE_EXT;
721 /* Note that, by the spec, the number of channels is (num_dbch + 1) */
722 e->num_chans =
723 readl_relaxed_bitmask(&mhu->ctrl->dbch_cfg0, num_dbch) + 1;
724 e->mbox_of_xlate = mhuv3_dbe_mbox_of_xlate;
725 e->combined_irq_setup = mhuv3_dbe_combined_irq_setup;
726 e->channels_init = mhuv3_dbe_channels_init;
727 e->chan_from_comb_irq_get = mhuv3_dbe_chan_from_comb_irq_get;
728
729 mhu->num_chans += e->num_chans * MHUV3_FLAG_BITS;
730 mhu->ext[DBE_EXT] = e;
731
732 dev_dbg(dev, "%s: found %d DBE channels.\n",
733 mhuv3_str[mhu->frame], e->num_chans);
734
735 return 0;
736}
737
738static int mhuv3_fce_init(struct mhuv3 *mhu)
739{
740 struct device *dev = mhu->mbox.dev;
741
742 if (!readl_relaxed_bitmask(&mhu->ctrl->feat_spt0, fce_spt))
743 return 0;
744
745 dev_dbg(dev, "%s: FCE Extension not supported by driver.\n",
746 mhuv3_str[mhu->frame]);
747
748 return 0;
749}
750
751static int mhuv3_fe_init(struct mhuv3 *mhu)
752{
753 struct device *dev = mhu->mbox.dev;
754
755 if (!readl_relaxed_bitmask(&mhu->ctrl->feat_spt0, fe_spt))
756 return 0;
757
758 dev_dbg(dev, "%s: FE Extension not supported by driver.\n",
759 mhuv3_str[mhu->frame]);
760
761 return 0;
762}
763
764static mhuv3_extension_initializer mhuv3_extension_init[NUM_EXT] = {
765 mhuv3_dbe_init,
766 mhuv3_fce_init,
767 mhuv3_fe_init,
768};
769
770static int mhuv3_initialize_channels(struct device *dev, struct mhuv3 *mhu)
771{
772 struct mbox_controller *mbox = &mhu->mbox;
773 int i, ret = 0;
774
775 mbox->chans = devm_kcalloc(dev, n: mhu->num_chans,
776 size: sizeof(*mbox->chans), GFP_KERNEL);
777 if (!mbox->chans)
778 return dev_err_probe(dev, err: -ENOMEM,
779 fmt: "Failed to initialize channels\n");
780
781 for (i = 0; i < NUM_EXT && !ret; i++)
782 if (mhu->ext[i])
783 ret = mhu->ext[i]->channels_init(mhu);
784
785 return ret;
786}
787
788static struct mbox_chan *mhuv3_mbox_of_xlate(struct mbox_controller *mbox,
789 const struct of_phandle_args *pa)
790{
791 struct mhuv3 *mhu = mhu_from_mbox(mbox);
792 unsigned int type, channel, param;
793
794 if (pa->args_count != MHUV3_MBOX_CELLS)
795 return ERR_PTR(error: -EINVAL);
796
797 type = pa->args[MHUV3_MBOX_CELL_TYPE];
798 if (type >= NUM_EXT)
799 return ERR_PTR(error: -EINVAL);
800
801 channel = pa->args[MHUV3_MBOX_CELL_CHWN];
802 param = pa->args[MHUV3_MBOX_CELL_PARAM];
803
804 return mhu->ext[type]->mbox_of_xlate(mhu, channel, param);
805}
806
807static void mhu_frame_cleanup_actions(void *data)
808{
809 struct mhuv3 *mhu = data;
810
811 writel_relaxed_bitmask(0x0, &mhu->ctrl->x_ctrl, op_req);
812}
813
814static int mhuv3_frame_init(struct mhuv3 *mhu, void __iomem *regs)
815{
816 struct device *dev = mhu->mbox.dev;
817 int i;
818
819 mhu->ctrl = regs;
820 mhu->frame = readl_relaxed_bitmask(&mhu->ctrl->blk_id, id);
821 if (mhu->frame > MBX_FRAME)
822 return dev_err_probe(dev, err: -EINVAL,
823 fmt: "Invalid Frame type- %d\n", mhu->frame);
824
825 mhu->major = readl_relaxed_bitmask(&mhu->ctrl->aidr, arch_major_rev);
826 mhu->minor = readl_relaxed_bitmask(&mhu->ctrl->aidr, arch_minor_rev);
827 mhu->implem = readl_relaxed_bitmask(&mhu->ctrl->iidr, implementer);
828 mhu->rev = readl_relaxed_bitmask(&mhu->ctrl->iidr, revision);
829 mhu->var = readl_relaxed_bitmask(&mhu->ctrl->iidr, variant);
830 mhu->prod_id = readl_relaxed_bitmask(&mhu->ctrl->iidr, product_id);
831 if (mhu->major != MHUV3_MAJOR_VERSION)
832 return dev_err_probe(dev, err: -EINVAL,
833 fmt: "Unsupported MHU %s block - major:%d minor:%d\n",
834 mhuv3_str[mhu->frame], mhu->major,
835 mhu->minor);
836
837 mhu->auto_op_full =
838 !!readl_relaxed_bitmask(&mhu->ctrl->feat_spt1, auto_op_spt);
839 /* Request the PBX/MBX to remain operational */
840 if (mhu->auto_op_full) {
841 writel_relaxed_bitmask(0x1, &mhu->ctrl->x_ctrl, op_req);
842 devm_add_action_or_reset(dev, mhu_frame_cleanup_actions, mhu);
843 }
844
845 dev_dbg(dev,
846 "Found MHU %s block - major:%d minor:%d\n implem:0x%X rev:0x%X var:0x%X prod_id:0x%X",
847 mhuv3_str[mhu->frame], mhu->major, mhu->minor,
848 mhu->implem, mhu->rev, mhu->var, mhu->prod_id);
849
850 if (mhu->frame == PBX_FRAME)
851 mhu->pbx = regs;
852 else
853 mhu->mbx = regs;
854
855 for (i = 0; i < NUM_EXT; i++) {
856 int ret;
857
858 /*
859 * Note that extensions initialization fails only when such
860 * extension initialization routine fails and the extensions
861 * was found to be supported in hardware and in software.
862 */
863 ret = mhuv3_extension_init[i](mhu);
864 if (ret)
865 return dev_err_probe(dev, err: ret,
866 fmt: "Failed to initialize %s %s\n",
867 mhuv3_str[mhu->frame],
868 mhuv3_ext_str[i]);
869 }
870
871 return 0;
872}
873
874static irqreturn_t mhuv3_pbx_comb_interrupt(int irq, void *arg)
875{
876 unsigned int i, found = 0;
877 struct mhuv3 *mhu = arg;
878 struct mbox_chan *chan;
879 struct device *dev;
880 int ret = IRQ_NONE;
881
882 dev = mhu->mbox.dev;
883 for (i = 0; i < NUM_EXT; i++) {
884 struct mhuv3_mbox_chan_priv *priv;
885
886 /* FCE does not participate to the PBX combined */
887 if (i == FCE_EXT || !mhu->ext[i])
888 continue;
889
890 chan = mhu->ext[i]->chan_from_comb_irq_get(mhu);
891 if (IS_ERR(ptr: chan))
892 continue;
893
894 found++;
895 priv = chan->con_priv;
896 if (!chan->cl) {
897 dev_warn(dev, "TX Ack on UNBOUND channel (%u)\n",
898 priv->ch_idx);
899 continue;
900 }
901
902 mbox_chan_txdone(chan, r: 0);
903 ret = IRQ_HANDLED;
904 }
905
906 if (found == 0)
907 dev_warn_once(dev, "Failed to find channel for the TX interrupt\n");
908
909 return ret;
910}
911
912static irqreturn_t mhuv3_mbx_comb_interrupt(int irq, void *arg)
913{
914 unsigned int i, found = 0;
915 struct mhuv3 *mhu = arg;
916 struct mbox_chan *chan;
917 struct device *dev;
918 int ret = IRQ_NONE;
919
920 dev = mhu->mbox.dev;
921 for (i = 0; i < NUM_EXT; i++) {
922 struct mhuv3_mbox_chan_priv *priv;
923 void *data __free(kfree) = NULL;
924
925 if (!mhu->ext[i])
926 continue;
927
928 /* Process any extension which could be source of the IRQ */
929 chan = mhu->ext[i]->chan_from_comb_irq_get(mhu);
930 if (IS_ERR(ptr: chan))
931 continue;
932
933 found++;
934 /* From here on we need to call rx_complete even on error */
935 priv = chan->con_priv;
936 if (!chan->cl) {
937 dev_warn(dev, "RX Data on UNBOUND channel (%u)\n",
938 priv->ch_idx);
939 goto rx_ack;
940 }
941
942 /* Read optional in-band LE data first. */
943 if (priv->ops->read_data) {
944 data = priv->ops->read_data(mhu, chan);
945 if (IS_ERR(ptr: data)) {
946 dev_err(dev,
947 "Failed to read in-band data. err:%ld\n",
948 PTR_ERR(no_free_ptr(data)));
949 goto rx_ack;
950 }
951 }
952
953 mbox_chan_received_data(chan, data);
954 ret = IRQ_HANDLED;
955
956 /*
957 * Acknowledge transfer after any possible optional
958 * out-of-band data has also been retrieved via
959 * mbox_chan_received_data().
960 */
961rx_ack:
962 if (priv->ops->rx_complete)
963 priv->ops->rx_complete(mhu, chan);
964 }
965
966 if (found == 0)
967 dev_warn_once(dev, "Failed to find channel for the RX interrupt\n");
968
969 return ret;
970}
971
972static int mhuv3_setup_pbx(struct mhuv3 *mhu)
973{
974 struct device *dev = mhu->mbox.dev;
975
976 mhu->mbox.ops = &mhuv3_sender_ops;
977
978 if (mhu->cmb_irq > 0) {
979 int ret, i;
980
981 ret = devm_request_threaded_irq(dev, irq: mhu->cmb_irq, NULL,
982 thread_fn: mhuv3_pbx_comb_interrupt,
983 IRQF_ONESHOT, devname: "mhuv3-pbx", dev_id: mhu);
984 if (ret)
985 return dev_err_probe(dev, err: ret,
986 fmt: "Failed to request PBX IRQ\n");
987
988 mhu->mbox.txdone_irq = true;
989 mhu->mbox.txdone_poll = false;
990
991 for (i = 0; i < NUM_EXT; i++)
992 if (mhu->ext[i])
993 mhu->ext[i]->combined_irq_setup(mhu);
994
995 dev_dbg(dev, "MHUv3 PBX IRQs initialized.\n");
996
997 return 0;
998 }
999
1000 dev_info(dev, "Using PBX in Tx polling mode.\n");
1001 mhu->mbox.txdone_irq = false;
1002 mhu->mbox.txdone_poll = true;
1003 mhu->mbox.txpoll_period = 1;
1004
1005 return 0;
1006}
1007
1008static int mhuv3_setup_mbx(struct mhuv3 *mhu)
1009{
1010 struct device *dev = mhu->mbox.dev;
1011 int ret, i;
1012
1013 mhu->mbox.ops = &mhuv3_receiver_ops;
1014
1015 if (mhu->cmb_irq <= 0)
1016 return dev_err_probe(dev, err: -EINVAL,
1017 fmt: "MBX combined IRQ is missing !\n");
1018
1019 ret = devm_request_threaded_irq(dev, irq: mhu->cmb_irq, NULL,
1020 thread_fn: mhuv3_mbx_comb_interrupt, IRQF_ONESHOT,
1021 devname: "mhuv3-mbx", dev_id: mhu);
1022 if (ret)
1023 return dev_err_probe(dev, err: ret, fmt: "Failed to request MBX IRQ\n");
1024
1025 for (i = 0; i < NUM_EXT; i++)
1026 if (mhu->ext[i])
1027 mhu->ext[i]->combined_irq_setup(mhu);
1028
1029 dev_dbg(dev, "MHUv3 MBX IRQs initialized.\n");
1030
1031 return ret;
1032}
1033
1034static int mhuv3_irqs_init(struct mhuv3 *mhu, struct platform_device *pdev)
1035{
1036 dev_dbg(mhu->mbox.dev, "Initializing %s block.\n",
1037 mhuv3_str[mhu->frame]);
1038
1039 if (mhu->frame == PBX_FRAME) {
1040 mhu->cmb_irq =
1041 platform_get_irq_byname_optional(dev: pdev, name: "combined");
1042 return mhuv3_setup_pbx(mhu);
1043 }
1044
1045 mhu->cmb_irq = platform_get_irq_byname(pdev, "combined");
1046 return mhuv3_setup_mbx(mhu);
1047}
1048
1049static int mhuv3_probe(struct platform_device *pdev)
1050{
1051 struct device *dev = &pdev->dev;
1052 void __iomem *regs;
1053 struct mhuv3 *mhu;
1054 int ret;
1055
1056 mhu = devm_kzalloc(dev, size: sizeof(*mhu), GFP_KERNEL);
1057 if (!mhu)
1058 return -ENOMEM;
1059
1060 regs = devm_platform_ioremap_resource(pdev, index: 0);
1061 if (IS_ERR(ptr: regs))
1062 return PTR_ERR(ptr: regs);
1063
1064 mhu->mbox.dev = dev;
1065 ret = mhuv3_frame_init(mhu, regs);
1066 if (ret)
1067 return ret;
1068
1069 ret = mhuv3_irqs_init(mhu, pdev);
1070 if (ret)
1071 return ret;
1072
1073 mhu->mbox.of_xlate = mhuv3_mbox_of_xlate;
1074 ret = mhuv3_initialize_channels(dev, mhu);
1075 if (ret)
1076 return ret;
1077
1078 ret = devm_mbox_controller_register(dev, mbox: &mhu->mbox);
1079 if (ret)
1080 return dev_err_probe(dev, err: ret,
1081 fmt: "Failed to register ARM MHUv3 driver\n");
1082
1083 return ret;
1084}
1085
1086static const struct of_device_id mhuv3_of_match[] = {
1087 { .compatible = "arm,mhuv3", .data = NULL },
1088 {}
1089};
1090MODULE_DEVICE_TABLE(of, mhuv3_of_match);
1091
1092static struct platform_driver mhuv3_driver = {
1093 .driver = {
1094 .name = "arm-mhuv3-mailbox",
1095 .of_match_table = mhuv3_of_match,
1096 },
1097 .probe = mhuv3_probe,
1098};
1099module_platform_driver(mhuv3_driver);
1100
1101MODULE_LICENSE("GPL");
1102MODULE_DESCRIPTION("ARM MHUv3 Driver");
1103MODULE_AUTHOR("Cristian Marussi <cristian.marussi@arm.com>");
1104

source code of linux/drivers/mailbox/arm_mhuv3.c