1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * ARM Message Handling Unit Version 2 (MHUv2) driver. |
4 | * |
5 | * Copyright (C) 2020 ARM Ltd. |
6 | * Copyright (C) 2020 Linaro Ltd. |
7 | * |
8 | * An MHUv2 mailbox controller can provide up to 124 channel windows (each 32 |
9 | * bit long) and the driver allows any combination of both the transport |
10 | * protocol modes: data-transfer and doorbell, to be used on those channel |
11 | * windows. |
12 | * |
13 | * The transport protocols should be specified in the device tree entry for the |
14 | * device. The transport protocols determine how the underlying hardware |
15 | * resources of the device are utilized when transmitting data. Refer to the |
16 | * device tree bindings of the ARM MHUv2 controller for more details. |
17 | * |
18 | * The number of registered mailbox channels is dependent on both the underlying |
19 | * hardware - mainly the number of channel windows implemented by the platform, |
20 | * as well as the selected transport protocols. |
21 | * |
22 | * The MHUv2 controller can work both as a sender and receiver, but the driver |
23 | * and the DT bindings support unidirectional transfers for better allocation of |
24 | * the channels. That is, this driver will be probed for two separate devices |
25 | * for each mailbox controller, a sender device and a receiver device. |
26 | */ |
27 | |
28 | #include <linux/amba/bus.h> |
29 | #include <linux/interrupt.h> |
30 | #include <linux/mailbox_controller.h> |
31 | #include <linux/mailbox/arm_mhuv2_message.h> |
32 | #include <linux/module.h> |
33 | #include <linux/of_address.h> |
34 | #include <linux/spinlock.h> |
35 | |
36 | /* ====== MHUv2 Registers ====== */ |
37 | |
38 | /* Maximum number of channel windows */ |
39 | #define MHUV2_CH_WN_MAX 124 |
40 | /* Number of combined interrupt status registers */ |
41 | #define MHUV2_CMB_INT_ST_REG_CNT 4 |
42 | #define MHUV2_STAT_BYTES (sizeof(u32)) |
43 | #define MHUV2_STAT_BITS (MHUV2_STAT_BYTES * __CHAR_BIT__) |
44 | |
45 | #define LSB_MASK(n) ((1 << (n * __CHAR_BIT__)) - 1) |
46 | #define MHUV2_PROTOCOL_PROP "arm,mhuv2-protocols" |
47 | |
48 | /* Register Message Handling Unit Configuration fields */ |
49 | struct mhu_cfg_t { |
50 | u32 num_ch : 7; |
51 | u32 pad : 25; |
52 | } __packed; |
53 | |
54 | /* register Interrupt Status fields */ |
55 | struct int_st_t { |
56 | u32 nr2r : 1; |
57 | u32 r2nr : 1; |
58 | u32 pad : 30; |
59 | } __packed; |
60 | |
61 | /* Register Interrupt Clear fields */ |
62 | struct int_clr_t { |
63 | u32 nr2r : 1; |
64 | u32 r2nr : 1; |
65 | u32 pad : 30; |
66 | } __packed; |
67 | |
68 | /* Register Interrupt Enable fields */ |
69 | struct int_en_t { |
70 | u32 r2nr : 1; |
71 | u32 nr2r : 1; |
72 | u32 chcomb : 1; |
73 | u32 pad : 29; |
74 | } __packed; |
75 | |
76 | /* Register Implementer Identification fields */ |
77 | struct iidr_t { |
78 | u32 implementer : 12; |
79 | u32 revision : 4; |
80 | u32 variant : 4; |
81 | u32 product_id : 12; |
82 | } __packed; |
83 | |
84 | /* Register Architecture Identification Register fields */ |
85 | struct aidr_t { |
86 | u32 arch_minor_rev : 4; |
87 | u32 arch_major_rev : 4; |
88 | u32 pad : 24; |
89 | } __packed; |
90 | |
91 | /* Sender Channel Window fields */ |
92 | struct mhu2_send_ch_wn_reg { |
93 | u32 stat; |
94 | u8 pad1[0x0C - 0x04]; |
95 | u32 stat_set; |
96 | u32 int_st; |
97 | u32 int_clr; |
98 | u32 int_en; |
99 | u8 pad2[0x20 - 0x1C]; |
100 | } __packed; |
101 | |
102 | /* Sender frame register fields */ |
103 | struct mhu2_send_frame_reg { |
104 | struct mhu2_send_ch_wn_reg ch_wn[MHUV2_CH_WN_MAX]; |
105 | struct mhu_cfg_t mhu_cfg; |
106 | u32 resp_cfg; |
107 | u32 access_request; |
108 | u32 access_ready; |
109 | struct int_st_t int_st; |
110 | struct int_clr_t int_clr; |
111 | struct int_en_t int_en; |
112 | u32 reserved0; |
113 | u32 chcomb_int_st[MHUV2_CMB_INT_ST_REG_CNT]; |
114 | u8 pad[0xFC8 - 0xFB0]; |
115 | struct iidr_t iidr; |
116 | struct aidr_t aidr; |
117 | } __packed; |
118 | |
119 | /* Receiver Channel Window fields */ |
120 | struct mhu2_recv_ch_wn_reg { |
121 | u32 stat; |
122 | u32 stat_masked; |
123 | u32 stat_clear; |
124 | u8 reserved0[0x10 - 0x0C]; |
125 | u32 mask; |
126 | u32 mask_set; |
127 | u32 mask_clear; |
128 | u8 pad[0x20 - 0x1C]; |
129 | } __packed; |
130 | |
131 | /* Receiver frame register fields */ |
132 | struct mhu2_recv_frame_reg { |
133 | struct mhu2_recv_ch_wn_reg ch_wn[MHUV2_CH_WN_MAX]; |
134 | struct mhu_cfg_t mhu_cfg; |
135 | u8 reserved0[0xF90 - 0xF84]; |
136 | struct int_st_t int_st; |
137 | struct int_clr_t int_clr; |
138 | struct int_en_t int_en; |
139 | u32 pad; |
140 | u32 chcomb_int_st[MHUV2_CMB_INT_ST_REG_CNT]; |
141 | u8 reserved2[0xFC8 - 0xFB0]; |
142 | struct iidr_t iidr; |
143 | struct aidr_t aidr; |
144 | } __packed; |
145 | |
146 | |
147 | /* ====== MHUv2 data structures ====== */ |
148 | |
149 | enum mhuv2_transport_protocol { |
150 | DOORBELL = 0, |
151 | DATA_TRANSFER = 1 |
152 | }; |
153 | |
154 | enum mhuv2_frame { |
155 | RECEIVER_FRAME, |
156 | SENDER_FRAME |
157 | }; |
158 | |
159 | /** |
160 | * struct mhuv2 - MHUv2 mailbox controller data |
161 | * |
162 | * @mbox: Mailbox controller belonging to the MHU frame. |
163 | * @send: Base address of the register mapping region. |
164 | * @recv: Base address of the register mapping region. |
165 | * @frame: Frame type: RECEIVER_FRAME or SENDER_FRAME. |
166 | * @irq: Interrupt. |
167 | * @windows: Channel windows implemented by the platform. |
168 | * @minor: Minor version of the controller. |
169 | * @length: Length of the protocols array in bytes. |
170 | * @protocols: Raw protocol information, derived from device tree. |
171 | * @doorbell_pending_lock: spinlock required for correct operation of Tx |
172 | * interrupt for doorbells. |
173 | */ |
174 | struct mhuv2 { |
175 | struct mbox_controller mbox; |
176 | union { |
177 | struct mhu2_send_frame_reg __iomem *send; |
178 | struct mhu2_recv_frame_reg __iomem *recv; |
179 | }; |
180 | enum mhuv2_frame frame; |
181 | unsigned int irq; |
182 | unsigned int windows; |
183 | unsigned int minor; |
184 | unsigned int length; |
185 | u32 *protocols; |
186 | |
187 | spinlock_t doorbell_pending_lock; |
188 | }; |
189 | |
190 | #define mhu_from_mbox(_mbox) container_of(_mbox, struct mhuv2, mbox) |
191 | |
192 | /** |
193 | * struct mhuv2_protocol_ops - MHUv2 operations |
194 | * |
195 | * Each transport protocol must provide an implementation of the operations |
196 | * provided here. |
197 | * |
198 | * @rx_startup: Startup callback for receiver. |
199 | * @rx_shutdown: Shutdown callback for receiver. |
200 | * @read_data: Reads and clears newly available data. |
201 | * @tx_startup: Startup callback for receiver. |
202 | * @tx_shutdown: Shutdown callback for receiver. |
203 | * @last_tx_done: Report back if the last tx is completed or not. |
204 | * @send_data: Send data to the receiver. |
205 | */ |
206 | struct mhuv2_protocol_ops { |
207 | int (*rx_startup)(struct mhuv2 *mhu, struct mbox_chan *chan); |
208 | void (*rx_shutdown)(struct mhuv2 *mhu, struct mbox_chan *chan); |
209 | void *(*read_data)(struct mhuv2 *mhu, struct mbox_chan *chan); |
210 | |
211 | void (*tx_startup)(struct mhuv2 *mhu, struct mbox_chan *chan); |
212 | void (*tx_shutdown)(struct mhuv2 *mhu, struct mbox_chan *chan); |
213 | int (*last_tx_done)(struct mhuv2 *mhu, struct mbox_chan *chan); |
214 | int (*send_data)(struct mhuv2 *mhu, struct mbox_chan *chan, void *arg); |
215 | }; |
216 | |
217 | /* |
218 | * MHUv2 mailbox channel's private information |
219 | * |
220 | * @ops: protocol specific ops for the channel. |
221 | * @ch_wn_idx: Channel window index allocated to the channel. |
222 | * @windows: Total number of windows consumed by the channel, only relevant |
223 | * in DATA_TRANSFER protocol. |
224 | * @doorbell: Doorbell bit number within the ch_wn_idx window, only relevant |
225 | * in DOORBELL protocol. |
226 | * @pending: Flag indicating pending doorbell interrupt, only relevant in |
227 | * DOORBELL protocol. |
228 | */ |
229 | struct mhuv2_mbox_chan_priv { |
230 | const struct mhuv2_protocol_ops *ops; |
231 | u32 ch_wn_idx; |
232 | union { |
233 | u32 windows; |
234 | struct { |
235 | u32 doorbell; |
236 | u32 pending; |
237 | }; |
238 | }; |
239 | }; |
240 | |
241 | /* Macro for reading a bitfield within a physically mapped packed struct */ |
242 | #define readl_relaxed_bitfield(_regptr, _type, _field) \ |
243 | ({ \ |
244 | u32 _regval; \ |
245 | _regval = readl_relaxed((_regptr)); \ |
246 | (*(_type *)(&_regval))._field; \ |
247 | }) |
248 | |
249 | /* Macro for writing a bitfield within a physically mapped packed struct */ |
250 | #define writel_relaxed_bitfield(_value, _regptr, _type, _field) \ |
251 | ({ \ |
252 | u32 _regval; \ |
253 | _regval = readl_relaxed(_regptr); \ |
254 | (*(_type *)(&_regval))._field = _value; \ |
255 | writel_relaxed(_regval, _regptr); \ |
256 | }) |
257 | |
258 | |
259 | /* =================== Doorbell transport protocol operations =============== */ |
260 | |
261 | static int mhuv2_doorbell_rx_startup(struct mhuv2 *mhu, struct mbox_chan *chan) |
262 | { |
263 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
264 | |
265 | writel_relaxed(BIT(priv->doorbell), |
266 | &mhu->recv->ch_wn[priv->ch_wn_idx].mask_clear); |
267 | return 0; |
268 | } |
269 | |
270 | static void mhuv2_doorbell_rx_shutdown(struct mhuv2 *mhu, |
271 | struct mbox_chan *chan) |
272 | { |
273 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
274 | |
275 | writel_relaxed(BIT(priv->doorbell), |
276 | &mhu->recv->ch_wn[priv->ch_wn_idx].mask_set); |
277 | } |
278 | |
279 | static void *mhuv2_doorbell_read_data(struct mhuv2 *mhu, struct mbox_chan *chan) |
280 | { |
281 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
282 | |
283 | writel_relaxed(BIT(priv->doorbell), |
284 | &mhu->recv->ch_wn[priv->ch_wn_idx].stat_clear); |
285 | return NULL; |
286 | } |
287 | |
288 | static int mhuv2_doorbell_last_tx_done(struct mhuv2 *mhu, |
289 | struct mbox_chan *chan) |
290 | { |
291 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
292 | |
293 | return !(readl_relaxed(&mhu->send->ch_wn[priv->ch_wn_idx].stat) & |
294 | BIT(priv->doorbell)); |
295 | } |
296 | |
297 | static int mhuv2_doorbell_send_data(struct mhuv2 *mhu, struct mbox_chan *chan, |
298 | void *arg) |
299 | { |
300 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
301 | unsigned long flags; |
302 | |
303 | spin_lock_irqsave(&mhu->doorbell_pending_lock, flags); |
304 | |
305 | priv->pending = 1; |
306 | writel_relaxed(BIT(priv->doorbell), |
307 | &mhu->send->ch_wn[priv->ch_wn_idx].stat_set); |
308 | |
309 | spin_unlock_irqrestore(lock: &mhu->doorbell_pending_lock, flags); |
310 | |
311 | return 0; |
312 | } |
313 | |
314 | static const struct mhuv2_protocol_ops mhuv2_doorbell_ops = { |
315 | .rx_startup = mhuv2_doorbell_rx_startup, |
316 | .rx_shutdown = mhuv2_doorbell_rx_shutdown, |
317 | .read_data = mhuv2_doorbell_read_data, |
318 | .last_tx_done = mhuv2_doorbell_last_tx_done, |
319 | .send_data = mhuv2_doorbell_send_data, |
320 | }; |
321 | #define IS_PROTOCOL_DOORBELL(_priv) (_priv->ops == &mhuv2_doorbell_ops) |
322 | |
323 | /* ============= Data transfer transport protocol operations ================ */ |
324 | |
325 | static int mhuv2_data_transfer_rx_startup(struct mhuv2 *mhu, |
326 | struct mbox_chan *chan) |
327 | { |
328 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
329 | int i = priv->ch_wn_idx + priv->windows - 1; |
330 | |
331 | /* |
332 | * The protocol mandates that all but the last status register must be |
333 | * masked. |
334 | */ |
335 | writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_clear); |
336 | return 0; |
337 | } |
338 | |
339 | static void mhuv2_data_transfer_rx_shutdown(struct mhuv2 *mhu, |
340 | struct mbox_chan *chan) |
341 | { |
342 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
343 | int i = priv->ch_wn_idx + priv->windows - 1; |
344 | |
345 | writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set); |
346 | } |
347 | |
348 | static void *mhuv2_data_transfer_read_data(struct mhuv2 *mhu, |
349 | struct mbox_chan *chan) |
350 | { |
351 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
352 | const int windows = priv->windows; |
353 | struct arm_mhuv2_mbox_msg *msg; |
354 | u32 *data; |
355 | int i, idx; |
356 | |
357 | msg = kzalloc(size: sizeof(*msg) + windows * MHUV2_STAT_BYTES, GFP_KERNEL); |
358 | if (!msg) |
359 | return ERR_PTR(error: -ENOMEM); |
360 | |
361 | data = msg->data = msg + 1; |
362 | msg->len = windows * MHUV2_STAT_BYTES; |
363 | |
364 | /* |
365 | * Messages are expected in order of most significant word to least |
366 | * significant word. Refer mhuv2_data_transfer_send_data() for more |
367 | * details. |
368 | * |
369 | * We also need to read the stat register instead of stat_masked, as we |
370 | * masked all but the last window. |
371 | * |
372 | * Last channel window must be cleared as the final operation. Upon |
373 | * clearing the last channel window register, which is unmasked in |
374 | * data-transfer protocol, the interrupt is de-asserted. |
375 | */ |
376 | for (i = 0; i < windows; i++) { |
377 | idx = priv->ch_wn_idx + i; |
378 | data[windows - 1 - i] = readl_relaxed(&mhu->recv->ch_wn[idx].stat); |
379 | writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[idx].stat_clear); |
380 | } |
381 | |
382 | return msg; |
383 | } |
384 | |
385 | static void mhuv2_data_transfer_tx_startup(struct mhuv2 *mhu, |
386 | struct mbox_chan *chan) |
387 | { |
388 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
389 | int i = priv->ch_wn_idx + priv->windows - 1; |
390 | |
391 | /* Enable interrupts only for the last window */ |
392 | if (mhu->minor) { |
393 | writel_relaxed(0x1, &mhu->send->ch_wn[i].int_clr); |
394 | writel_relaxed(0x1, &mhu->send->ch_wn[i].int_en); |
395 | } |
396 | } |
397 | |
398 | static void mhuv2_data_transfer_tx_shutdown(struct mhuv2 *mhu, |
399 | struct mbox_chan *chan) |
400 | { |
401 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
402 | int i = priv->ch_wn_idx + priv->windows - 1; |
403 | |
404 | if (mhu->minor) |
405 | writel_relaxed(0x0, &mhu->send->ch_wn[i].int_en); |
406 | } |
407 | |
408 | static int mhuv2_data_transfer_last_tx_done(struct mhuv2 *mhu, |
409 | struct mbox_chan *chan) |
410 | { |
411 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
412 | int i = priv->ch_wn_idx + priv->windows - 1; |
413 | |
414 | /* Just checking the last channel window should be enough */ |
415 | return !readl_relaxed(&mhu->send->ch_wn[i].stat); |
416 | } |
417 | |
418 | /* |
419 | * Message will be transmitted from most significant to least significant word. |
420 | * This is to allow for messages shorter than channel windows to still trigger |
421 | * the receiver interrupt which gets activated when the last stat register is |
422 | * written. As an example, a 6-word message is to be written on a 4-channel MHU |
423 | * connection: Registers marked with '*' are masked, and will not generate an |
424 | * interrupt on the receiver side once written. |
425 | * |
426 | * u32 *data = [0x00000001], [0x00000002], [0x00000003], [0x00000004], |
427 | * [0x00000005], [0x00000006] |
428 | * |
429 | * ROUND 1: |
430 | * stat reg To write Write sequence |
431 | * [ stat 3 ] <- [0x00000001] 4 <- triggers interrupt on receiver |
432 | * [ stat 2 ] <- [0x00000002] 3 |
433 | * [ stat 1 ] <- [0x00000003] 2 |
434 | * [ stat 0 ] <- [0x00000004] 1 |
435 | * |
436 | * data += 4 // Increment data pointer by number of stat regs |
437 | * |
438 | * ROUND 2: |
439 | * stat reg To write Write sequence |
440 | * [ stat 3 ] <- [0x00000005] 2 <- triggers interrupt on receiver |
441 | * [ stat 2 ] <- [0x00000006] 1 |
442 | * [ stat 1 ] <- [0x00000000] |
443 | * [ stat 0 ] <- [0x00000000] |
444 | */ |
445 | static int mhuv2_data_transfer_send_data(struct mhuv2 *mhu, |
446 | struct mbox_chan *chan, void *arg) |
447 | { |
448 | const struct arm_mhuv2_mbox_msg *msg = arg; |
449 | int bytes_left = msg->len, bytes_to_send, bytes_in_round, i; |
450 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
451 | int windows = priv->windows; |
452 | u32 *data = msg->data, word; |
453 | |
454 | while (bytes_left) { |
455 | if (!data[0]) { |
456 | dev_err(mhu->mbox.dev, "Data aligned at first window can't be zero to guarantee interrupt generation at receiver" ); |
457 | return -EINVAL; |
458 | } |
459 | |
460 | while(!mhuv2_data_transfer_last_tx_done(mhu, chan)) |
461 | continue; |
462 | |
463 | bytes_in_round = min(bytes_left, (int)(windows * MHUV2_STAT_BYTES)); |
464 | |
465 | for (i = windows - 1; i >= 0; i--) { |
466 | /* Data less than windows can transfer ? */ |
467 | if (unlikely(bytes_in_round <= i * MHUV2_STAT_BYTES)) |
468 | continue; |
469 | |
470 | word = data[i]; |
471 | bytes_to_send = bytes_in_round & (MHUV2_STAT_BYTES - 1); |
472 | if (unlikely(bytes_to_send)) |
473 | word &= LSB_MASK(bytes_to_send); |
474 | else |
475 | bytes_to_send = MHUV2_STAT_BYTES; |
476 | |
477 | writel_relaxed(word, &mhu->send->ch_wn[priv->ch_wn_idx + windows - 1 - i].stat_set); |
478 | bytes_left -= bytes_to_send; |
479 | bytes_in_round -= bytes_to_send; |
480 | } |
481 | |
482 | data += windows; |
483 | } |
484 | |
485 | return 0; |
486 | } |
487 | |
488 | static const struct mhuv2_protocol_ops mhuv2_data_transfer_ops = { |
489 | .rx_startup = mhuv2_data_transfer_rx_startup, |
490 | .rx_shutdown = mhuv2_data_transfer_rx_shutdown, |
491 | .read_data = mhuv2_data_transfer_read_data, |
492 | .tx_startup = mhuv2_data_transfer_tx_startup, |
493 | .tx_shutdown = mhuv2_data_transfer_tx_shutdown, |
494 | .last_tx_done = mhuv2_data_transfer_last_tx_done, |
495 | .send_data = mhuv2_data_transfer_send_data, |
496 | }; |
497 | |
498 | /* Interrupt handlers */ |
499 | |
500 | static struct mbox_chan *get_irq_chan_comb(struct mhuv2 *mhu, u32 __iomem *reg) |
501 | { |
502 | struct mbox_chan *chans = mhu->mbox.chans; |
503 | int channel = 0, i, offset = 0, windows, protocol, ch_wn; |
504 | u32 stat; |
505 | |
506 | for (i = 0; i < MHUV2_CMB_INT_ST_REG_CNT; i++) { |
507 | stat = readl_relaxed(reg + i); |
508 | if (!stat) |
509 | continue; |
510 | |
511 | ch_wn = i * MHUV2_STAT_BITS + __builtin_ctz(stat); |
512 | |
513 | for (i = 0; i < mhu->length; i += 2) { |
514 | protocol = mhu->protocols[i]; |
515 | windows = mhu->protocols[i + 1]; |
516 | |
517 | if (ch_wn >= offset + windows) { |
518 | if (protocol == DOORBELL) |
519 | channel += MHUV2_STAT_BITS * windows; |
520 | else |
521 | channel++; |
522 | |
523 | offset += windows; |
524 | continue; |
525 | } |
526 | |
527 | /* Return first chan of the window in doorbell mode */ |
528 | if (protocol == DOORBELL) |
529 | channel += MHUV2_STAT_BITS * (ch_wn - offset); |
530 | |
531 | return &chans[channel]; |
532 | } |
533 | } |
534 | |
535 | return ERR_PTR(error: -EIO); |
536 | } |
537 | |
538 | static irqreturn_t mhuv2_sender_interrupt(int irq, void *data) |
539 | { |
540 | struct mhuv2 *mhu = data; |
541 | struct device *dev = mhu->mbox.dev; |
542 | struct mhuv2_mbox_chan_priv *priv; |
543 | struct mbox_chan *chan; |
544 | unsigned long flags; |
545 | int i, found = 0; |
546 | u32 stat; |
547 | |
548 | chan = get_irq_chan_comb(mhu, reg: mhu->send->chcomb_int_st); |
549 | if (IS_ERR(ptr: chan)) { |
550 | dev_warn(dev, "Failed to find channel for the Tx interrupt\n" ); |
551 | return IRQ_NONE; |
552 | } |
553 | priv = chan->con_priv; |
554 | |
555 | if (!IS_PROTOCOL_DOORBELL(priv)) { |
556 | for (i = 0; i < priv->windows; i++) |
557 | writel_relaxed(1, &mhu->send->ch_wn[priv->ch_wn_idx + i].int_clr); |
558 | |
559 | if (chan->cl) { |
560 | mbox_chan_txdone(chan, r: 0); |
561 | return IRQ_HANDLED; |
562 | } |
563 | |
564 | dev_warn(dev, "Tx interrupt Received on channel (%u) not currently attached to a mailbox client\n" , |
565 | priv->ch_wn_idx); |
566 | return IRQ_NONE; |
567 | } |
568 | |
569 | /* Clear the interrupt first, so we don't miss any doorbell later */ |
570 | writel_relaxed(1, &mhu->send->ch_wn[priv->ch_wn_idx].int_clr); |
571 | |
572 | /* |
573 | * In Doorbell mode, make sure no new transitions happen while the |
574 | * interrupt handler is trying to find the finished doorbell tx |
575 | * operations, else we may think few of the transfers were complete |
576 | * before they actually were. |
577 | */ |
578 | spin_lock_irqsave(&mhu->doorbell_pending_lock, flags); |
579 | |
580 | /* |
581 | * In case of doorbell mode, the first channel of the window is returned |
582 | * by get_irq_chan_comb(). Find all the pending channels here. |
583 | */ |
584 | stat = readl_relaxed(&mhu->send->ch_wn[priv->ch_wn_idx].stat); |
585 | |
586 | for (i = 0; i < MHUV2_STAT_BITS; i++) { |
587 | priv = chan[i].con_priv; |
588 | |
589 | /* Find cases where pending was 1, but stat's bit is cleared */ |
590 | if (priv->pending ^ ((stat >> i) & 0x1)) { |
591 | BUG_ON(!priv->pending); |
592 | |
593 | if (!chan->cl) { |
594 | dev_warn(dev, "Tx interrupt received on doorbell (%u : %u) channel not currently attached to a mailbox client\n" , |
595 | priv->ch_wn_idx, i); |
596 | continue; |
597 | } |
598 | |
599 | mbox_chan_txdone(chan: &chan[i], r: 0); |
600 | priv->pending = 0; |
601 | found++; |
602 | } |
603 | } |
604 | |
605 | spin_unlock_irqrestore(lock: &mhu->doorbell_pending_lock, flags); |
606 | |
607 | if (!found) { |
608 | /* |
609 | * We may have already processed the doorbell in the previous |
610 | * iteration if the interrupt came right after we cleared it but |
611 | * before we read the stat register. |
612 | */ |
613 | dev_dbg(dev, "Couldn't find the doorbell (%u) for the Tx interrupt interrupt\n" , |
614 | priv->ch_wn_idx); |
615 | return IRQ_NONE; |
616 | } |
617 | |
618 | return IRQ_HANDLED; |
619 | } |
620 | |
621 | static struct mbox_chan *get_irq_chan_comb_rx(struct mhuv2 *mhu) |
622 | { |
623 | struct mhuv2_mbox_chan_priv *priv; |
624 | struct mbox_chan *chan; |
625 | u32 stat; |
626 | |
627 | chan = get_irq_chan_comb(mhu, reg: mhu->recv->chcomb_int_st); |
628 | if (IS_ERR(ptr: chan)) |
629 | return chan; |
630 | |
631 | priv = chan->con_priv; |
632 | if (!IS_PROTOCOL_DOORBELL(priv)) |
633 | return chan; |
634 | |
635 | /* |
636 | * In case of doorbell mode, the first channel of the window is returned |
637 | * by the routine. Find the exact channel here. |
638 | */ |
639 | stat = readl_relaxed(&mhu->recv->ch_wn[priv->ch_wn_idx].stat_masked); |
640 | BUG_ON(!stat); |
641 | |
642 | return chan + __builtin_ctz(stat); |
643 | } |
644 | |
645 | static struct mbox_chan *get_irq_chan_stat_rx(struct mhuv2 *mhu) |
646 | { |
647 | struct mbox_chan *chans = mhu->mbox.chans; |
648 | struct mhuv2_mbox_chan_priv *priv; |
649 | u32 stat; |
650 | int i = 0; |
651 | |
652 | while (i < mhu->mbox.num_chans) { |
653 | priv = chans[i].con_priv; |
654 | stat = readl_relaxed(&mhu->recv->ch_wn[priv->ch_wn_idx].stat_masked); |
655 | |
656 | if (stat) { |
657 | if (IS_PROTOCOL_DOORBELL(priv)) |
658 | i += __builtin_ctz(stat); |
659 | return &chans[i]; |
660 | } |
661 | |
662 | i += IS_PROTOCOL_DOORBELL(priv) ? MHUV2_STAT_BITS : 1; |
663 | } |
664 | |
665 | return ERR_PTR(error: -EIO); |
666 | } |
667 | |
668 | static struct mbox_chan *get_irq_chan_rx(struct mhuv2 *mhu) |
669 | { |
670 | if (!mhu->minor) |
671 | return get_irq_chan_stat_rx(mhu); |
672 | |
673 | return get_irq_chan_comb_rx(mhu); |
674 | } |
675 | |
676 | static irqreturn_t mhuv2_receiver_interrupt(int irq, void *arg) |
677 | { |
678 | struct mhuv2 *mhu = arg; |
679 | struct mbox_chan *chan = get_irq_chan_rx(mhu); |
680 | struct device *dev = mhu->mbox.dev; |
681 | struct mhuv2_mbox_chan_priv *priv; |
682 | int ret = IRQ_NONE; |
683 | void *data; |
684 | |
685 | if (IS_ERR(ptr: chan)) { |
686 | dev_warn(dev, "Failed to find channel for the rx interrupt\n" ); |
687 | return IRQ_NONE; |
688 | } |
689 | priv = chan->con_priv; |
690 | |
691 | /* Read and clear the data first */ |
692 | data = priv->ops->read_data(mhu, chan); |
693 | |
694 | if (!chan->cl) { |
695 | dev_warn(dev, "Received data on channel (%u) not currently attached to a mailbox client\n" , |
696 | priv->ch_wn_idx); |
697 | } else if (IS_ERR(ptr: data)) { |
698 | dev_err(dev, "Failed to read data: %lu\n" , PTR_ERR(data)); |
699 | } else { |
700 | mbox_chan_received_data(chan, data); |
701 | ret = IRQ_HANDLED; |
702 | } |
703 | |
704 | if (!IS_ERR(ptr: data)) |
705 | kfree(objp: data); |
706 | |
707 | return ret; |
708 | } |
709 | |
710 | /* Sender and receiver ops */ |
711 | static bool mhuv2_sender_last_tx_done(struct mbox_chan *chan) |
712 | { |
713 | struct mhuv2 *mhu = mhu_from_mbox(chan->mbox); |
714 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
715 | |
716 | return priv->ops->last_tx_done(mhu, chan); |
717 | } |
718 | |
719 | static int mhuv2_sender_send_data(struct mbox_chan *chan, void *data) |
720 | { |
721 | struct mhuv2 *mhu = mhu_from_mbox(chan->mbox); |
722 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
723 | |
724 | if (!priv->ops->last_tx_done(mhu, chan)) |
725 | return -EBUSY; |
726 | |
727 | return priv->ops->send_data(mhu, chan, data); |
728 | } |
729 | |
730 | static int mhuv2_sender_startup(struct mbox_chan *chan) |
731 | { |
732 | struct mhuv2 *mhu = mhu_from_mbox(chan->mbox); |
733 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
734 | |
735 | if (priv->ops->tx_startup) |
736 | priv->ops->tx_startup(mhu, chan); |
737 | return 0; |
738 | } |
739 | |
740 | static void mhuv2_sender_shutdown(struct mbox_chan *chan) |
741 | { |
742 | struct mhuv2 *mhu = mhu_from_mbox(chan->mbox); |
743 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
744 | |
745 | if (priv->ops->tx_shutdown) |
746 | priv->ops->tx_shutdown(mhu, chan); |
747 | } |
748 | |
749 | static const struct mbox_chan_ops mhuv2_sender_ops = { |
750 | .send_data = mhuv2_sender_send_data, |
751 | .startup = mhuv2_sender_startup, |
752 | .shutdown = mhuv2_sender_shutdown, |
753 | .last_tx_done = mhuv2_sender_last_tx_done, |
754 | }; |
755 | |
756 | static int mhuv2_receiver_startup(struct mbox_chan *chan) |
757 | { |
758 | struct mhuv2 *mhu = mhu_from_mbox(chan->mbox); |
759 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
760 | |
761 | return priv->ops->rx_startup(mhu, chan); |
762 | } |
763 | |
764 | static void mhuv2_receiver_shutdown(struct mbox_chan *chan) |
765 | { |
766 | struct mhuv2 *mhu = mhu_from_mbox(chan->mbox); |
767 | struct mhuv2_mbox_chan_priv *priv = chan->con_priv; |
768 | |
769 | priv->ops->rx_shutdown(mhu, chan); |
770 | } |
771 | |
772 | static int mhuv2_receiver_send_data(struct mbox_chan *chan, void *data) |
773 | { |
774 | dev_err(chan->mbox->dev, |
775 | "Trying to transmit on a receiver MHU frame\n" ); |
776 | return -EIO; |
777 | } |
778 | |
779 | static bool mhuv2_receiver_last_tx_done(struct mbox_chan *chan) |
780 | { |
781 | dev_err(chan->mbox->dev, "Trying to Tx poll on a receiver MHU frame\n" ); |
782 | return true; |
783 | } |
784 | |
785 | static const struct mbox_chan_ops mhuv2_receiver_ops = { |
786 | .send_data = mhuv2_receiver_send_data, |
787 | .startup = mhuv2_receiver_startup, |
788 | .shutdown = mhuv2_receiver_shutdown, |
789 | .last_tx_done = mhuv2_receiver_last_tx_done, |
790 | }; |
791 | |
792 | static struct mbox_chan *mhuv2_mbox_of_xlate(struct mbox_controller *mbox, |
793 | const struct of_phandle_args *pa) |
794 | { |
795 | struct mhuv2 *mhu = mhu_from_mbox(mbox); |
796 | struct mbox_chan *chans = mbox->chans; |
797 | int channel = 0, i, offset, doorbell, protocol, windows; |
798 | |
799 | if (pa->args_count != 2) |
800 | return ERR_PTR(error: -EINVAL); |
801 | |
802 | offset = pa->args[0]; |
803 | doorbell = pa->args[1]; |
804 | if (doorbell >= MHUV2_STAT_BITS) |
805 | goto out; |
806 | |
807 | for (i = 0; i < mhu->length; i += 2) { |
808 | protocol = mhu->protocols[i]; |
809 | windows = mhu->protocols[i + 1]; |
810 | |
811 | if (protocol == DOORBELL) { |
812 | if (offset < windows) |
813 | return &chans[channel + MHUV2_STAT_BITS * offset + doorbell]; |
814 | |
815 | channel += MHUV2_STAT_BITS * windows; |
816 | offset -= windows; |
817 | } else { |
818 | if (offset == 0) { |
819 | if (doorbell) |
820 | goto out; |
821 | |
822 | return &chans[channel]; |
823 | } |
824 | |
825 | channel++; |
826 | offset--; |
827 | } |
828 | } |
829 | |
830 | out: |
831 | dev_err(mbox->dev, "Couldn't xlate to a valid channel (%d: %d)\n" , |
832 | pa->args[0], doorbell); |
833 | return ERR_PTR(error: -ENODEV); |
834 | } |
835 | |
836 | static int mhuv2_verify_protocol(struct mhuv2 *mhu) |
837 | { |
838 | struct device *dev = mhu->mbox.dev; |
839 | int protocol, windows, channels = 0, total_windows = 0, i; |
840 | |
841 | for (i = 0; i < mhu->length; i += 2) { |
842 | protocol = mhu->protocols[i]; |
843 | windows = mhu->protocols[i + 1]; |
844 | |
845 | if (!windows) { |
846 | dev_err(dev, "Window size can't be zero (%d)\n" , i); |
847 | return -EINVAL; |
848 | } |
849 | total_windows += windows; |
850 | |
851 | if (protocol == DOORBELL) { |
852 | channels += MHUV2_STAT_BITS * windows; |
853 | } else if (protocol == DATA_TRANSFER) { |
854 | channels++; |
855 | } else { |
856 | dev_err(dev, "Invalid protocol (%d) present in %s property at index %d\n" , |
857 | protocol, MHUV2_PROTOCOL_PROP, i); |
858 | return -EINVAL; |
859 | } |
860 | } |
861 | |
862 | if (total_windows > mhu->windows) { |
863 | dev_err(dev, "Channel windows can't be more than what's implemented by the hardware ( %d: %d)\n" , |
864 | total_windows, mhu->windows); |
865 | return -EINVAL; |
866 | } |
867 | |
868 | mhu->mbox.num_chans = channels; |
869 | return 0; |
870 | } |
871 | |
872 | static int mhuv2_allocate_channels(struct mhuv2 *mhu) |
873 | { |
874 | struct mbox_controller *mbox = &mhu->mbox; |
875 | struct mhuv2_mbox_chan_priv *priv; |
876 | struct device *dev = mbox->dev; |
877 | struct mbox_chan *chans; |
878 | int protocol, windows = 0, next_window = 0, i, j, k; |
879 | |
880 | chans = devm_kcalloc(dev, n: mbox->num_chans, size: sizeof(*chans), GFP_KERNEL); |
881 | if (!chans) |
882 | return -ENOMEM; |
883 | |
884 | mbox->chans = chans; |
885 | |
886 | for (i = 0; i < mhu->length; i += 2) { |
887 | next_window += windows; |
888 | |
889 | protocol = mhu->protocols[i]; |
890 | windows = mhu->protocols[i + 1]; |
891 | |
892 | if (protocol == DATA_TRANSFER) { |
893 | priv = devm_kmalloc(dev, size: sizeof(*priv), GFP_KERNEL); |
894 | if (!priv) |
895 | return -ENOMEM; |
896 | |
897 | priv->ch_wn_idx = next_window; |
898 | priv->ops = &mhuv2_data_transfer_ops; |
899 | priv->windows = windows; |
900 | chans++->con_priv = priv; |
901 | continue; |
902 | } |
903 | |
904 | for (j = 0; j < windows; j++) { |
905 | for (k = 0; k < MHUV2_STAT_BITS; k++) { |
906 | priv = devm_kmalloc(dev, size: sizeof(*priv), GFP_KERNEL); |
907 | if (!priv) |
908 | return -ENOMEM; |
909 | |
910 | priv->ch_wn_idx = next_window + j; |
911 | priv->ops = &mhuv2_doorbell_ops; |
912 | priv->doorbell = k; |
913 | chans++->con_priv = priv; |
914 | } |
915 | |
916 | /* |
917 | * Permanently enable interrupt as we can't |
918 | * control it per doorbell. |
919 | */ |
920 | if (mhu->frame == SENDER_FRAME && mhu->minor) |
921 | writel_relaxed(0x1, &mhu->send->ch_wn[priv->ch_wn_idx].int_en); |
922 | } |
923 | } |
924 | |
925 | /* Make sure we have initialized all channels */ |
926 | BUG_ON(chans - mbox->chans != mbox->num_chans); |
927 | |
928 | return 0; |
929 | } |
930 | |
931 | static int mhuv2_parse_channels(struct mhuv2 *mhu) |
932 | { |
933 | struct device *dev = mhu->mbox.dev; |
934 | const struct device_node *np = dev->of_node; |
935 | int ret, count; |
936 | u32 *protocols; |
937 | |
938 | count = of_property_count_u32_elems(np, MHUV2_PROTOCOL_PROP); |
939 | if (count <= 0 || count % 2) { |
940 | dev_err(dev, "Invalid %s property (%d)\n" , MHUV2_PROTOCOL_PROP, |
941 | count); |
942 | return -EINVAL; |
943 | } |
944 | |
945 | protocols = devm_kmalloc_array(dev, n: count, size: sizeof(*protocols), GFP_KERNEL); |
946 | if (!protocols) |
947 | return -ENOMEM; |
948 | |
949 | ret = of_property_read_u32_array(np, MHUV2_PROTOCOL_PROP, out_values: protocols, sz: count); |
950 | if (ret) { |
951 | dev_err(dev, "Failed to read %s property: %d\n" , |
952 | MHUV2_PROTOCOL_PROP, ret); |
953 | return ret; |
954 | } |
955 | |
956 | mhu->protocols = protocols; |
957 | mhu->length = count; |
958 | |
959 | ret = mhuv2_verify_protocol(mhu); |
960 | if (ret) |
961 | return ret; |
962 | |
963 | return mhuv2_allocate_channels(mhu); |
964 | } |
965 | |
966 | static int mhuv2_tx_init(struct amba_device *adev, struct mhuv2 *mhu, |
967 | void __iomem *reg) |
968 | { |
969 | struct device *dev = mhu->mbox.dev; |
970 | int ret, i; |
971 | |
972 | mhu->frame = SENDER_FRAME; |
973 | mhu->mbox.ops = &mhuv2_sender_ops; |
974 | mhu->send = reg; |
975 | |
976 | mhu->windows = readl_relaxed_bitfield(&mhu->send->mhu_cfg, struct mhu_cfg_t, num_ch); |
977 | mhu->minor = readl_relaxed_bitfield(&mhu->send->aidr, struct aidr_t, arch_minor_rev); |
978 | |
979 | spin_lock_init(&mhu->doorbell_pending_lock); |
980 | |
981 | /* |
982 | * For minor version 1 and forward, tx interrupt is provided by |
983 | * the controller. |
984 | */ |
985 | if (mhu->minor && adev->irq[0]) { |
986 | ret = devm_request_threaded_irq(dev, irq: adev->irq[0], NULL, |
987 | thread_fn: mhuv2_sender_interrupt, |
988 | IRQF_ONESHOT, devname: "mhuv2-tx" , dev_id: mhu); |
989 | if (ret) { |
990 | dev_err(dev, "Failed to request tx IRQ, fallback to polling mode: %d\n" , |
991 | ret); |
992 | } else { |
993 | mhu->mbox.txdone_irq = true; |
994 | mhu->mbox.txdone_poll = false; |
995 | mhu->irq = adev->irq[0]; |
996 | |
997 | writel_relaxed_bitfield(1, &mhu->send->int_en, struct int_en_t, chcomb); |
998 | |
999 | /* Disable all channel interrupts */ |
1000 | for (i = 0; i < mhu->windows; i++) |
1001 | writel_relaxed(0x0, &mhu->send->ch_wn[i].int_en); |
1002 | |
1003 | goto out; |
1004 | } |
1005 | } |
1006 | |
1007 | mhu->mbox.txdone_irq = false; |
1008 | mhu->mbox.txdone_poll = true; |
1009 | mhu->mbox.txpoll_period = 1; |
1010 | |
1011 | out: |
1012 | /* Wait for receiver to be ready */ |
1013 | writel_relaxed(0x1, &mhu->send->access_request); |
1014 | while (!readl_relaxed(&mhu->send->access_ready)) |
1015 | continue; |
1016 | |
1017 | return 0; |
1018 | } |
1019 | |
1020 | static int mhuv2_rx_init(struct amba_device *adev, struct mhuv2 *mhu, |
1021 | void __iomem *reg) |
1022 | { |
1023 | struct device *dev = mhu->mbox.dev; |
1024 | int ret, i; |
1025 | |
1026 | mhu->frame = RECEIVER_FRAME; |
1027 | mhu->mbox.ops = &mhuv2_receiver_ops; |
1028 | mhu->recv = reg; |
1029 | |
1030 | mhu->windows = readl_relaxed_bitfield(&mhu->recv->mhu_cfg, struct mhu_cfg_t, num_ch); |
1031 | mhu->minor = readl_relaxed_bitfield(&mhu->recv->aidr, struct aidr_t, arch_minor_rev); |
1032 | |
1033 | mhu->irq = adev->irq[0]; |
1034 | if (!mhu->irq) { |
1035 | dev_err(dev, "Missing receiver IRQ\n" ); |
1036 | return -EINVAL; |
1037 | } |
1038 | |
1039 | ret = devm_request_threaded_irq(dev, irq: mhu->irq, NULL, |
1040 | thread_fn: mhuv2_receiver_interrupt, IRQF_ONESHOT, |
1041 | devname: "mhuv2-rx" , dev_id: mhu); |
1042 | if (ret) { |
1043 | dev_err(dev, "Failed to request rx IRQ\n" ); |
1044 | return ret; |
1045 | } |
1046 | |
1047 | /* Mask all the channel windows */ |
1048 | for (i = 0; i < mhu->windows; i++) |
1049 | writel_relaxed(0xFFFFFFFF, &mhu->recv->ch_wn[i].mask_set); |
1050 | |
1051 | if (mhu->minor) |
1052 | writel_relaxed_bitfield(1, &mhu->recv->int_en, struct int_en_t, chcomb); |
1053 | |
1054 | return 0; |
1055 | } |
1056 | |
1057 | static int mhuv2_probe(struct amba_device *adev, const struct amba_id *id) |
1058 | { |
1059 | struct device *dev = &adev->dev; |
1060 | const struct device_node *np = dev->of_node; |
1061 | struct mhuv2 *mhu; |
1062 | void __iomem *reg; |
1063 | int ret = -EINVAL; |
1064 | |
1065 | reg = devm_of_iomap(dev, node: dev->of_node, index: 0, NULL); |
1066 | if (IS_ERR(ptr: reg)) |
1067 | return PTR_ERR(ptr: reg); |
1068 | |
1069 | mhu = devm_kzalloc(dev, size: sizeof(*mhu), GFP_KERNEL); |
1070 | if (!mhu) |
1071 | return -ENOMEM; |
1072 | |
1073 | mhu->mbox.dev = dev; |
1074 | mhu->mbox.of_xlate = mhuv2_mbox_of_xlate; |
1075 | |
1076 | if (of_device_is_compatible(device: np, "arm,mhuv2-tx" )) |
1077 | ret = mhuv2_tx_init(adev, mhu, reg); |
1078 | else if (of_device_is_compatible(device: np, "arm,mhuv2-rx" )) |
1079 | ret = mhuv2_rx_init(adev, mhu, reg); |
1080 | else |
1081 | dev_err(dev, "Invalid compatible property\n" ); |
1082 | |
1083 | if (ret) |
1084 | return ret; |
1085 | |
1086 | /* Channel windows can't be 0 */ |
1087 | BUG_ON(!mhu->windows); |
1088 | |
1089 | ret = mhuv2_parse_channels(mhu); |
1090 | if (ret) |
1091 | return ret; |
1092 | |
1093 | amba_set_drvdata(adev, mhu); |
1094 | |
1095 | ret = devm_mbox_controller_register(dev, mbox: &mhu->mbox); |
1096 | if (ret) |
1097 | dev_err(dev, "failed to register ARM MHUv2 driver %d\n" , ret); |
1098 | |
1099 | return ret; |
1100 | } |
1101 | |
1102 | static void mhuv2_remove(struct amba_device *adev) |
1103 | { |
1104 | struct mhuv2 *mhu = amba_get_drvdata(adev); |
1105 | |
1106 | if (mhu->frame == SENDER_FRAME) |
1107 | writel_relaxed(0x0, &mhu->send->access_request); |
1108 | } |
1109 | |
1110 | static struct amba_id mhuv2_ids[] = { |
1111 | { |
1112 | /* 2.0 */ |
1113 | .id = 0xbb0d1, |
1114 | .mask = 0xfffff, |
1115 | }, |
1116 | { |
1117 | /* 2.1 */ |
1118 | .id = 0xbb076, |
1119 | .mask = 0xfffff, |
1120 | }, |
1121 | { 0, 0 }, |
1122 | }; |
1123 | MODULE_DEVICE_TABLE(amba, mhuv2_ids); |
1124 | |
1125 | static struct amba_driver mhuv2_driver = { |
1126 | .drv = { |
1127 | .name = "arm-mhuv2" , |
1128 | }, |
1129 | .id_table = mhuv2_ids, |
1130 | .probe = mhuv2_probe, |
1131 | .remove = mhuv2_remove, |
1132 | }; |
1133 | module_amba_driver(mhuv2_driver); |
1134 | |
1135 | MODULE_LICENSE("GPL v2" ); |
1136 | MODULE_DESCRIPTION("ARM MHUv2 Driver" ); |
1137 | MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>" ); |
1138 | MODULE_AUTHOR("Tushar Khandelwal <tushar.khandelwal@arm.com>" ); |
1139 | |