1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2016-2023, NVIDIA CORPORATION. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/delay.h> |
7 | #include <linux/interrupt.h> |
8 | #include <linux/io.h> |
9 | #include <linux/mailbox_controller.h> |
10 | #include <linux/of.h> |
11 | #include <linux/platform_device.h> |
12 | #include <linux/pm.h> |
13 | #include <linux/slab.h> |
14 | |
15 | #include <soc/tegra/fuse.h> |
16 | |
17 | #include <dt-bindings/mailbox/tegra186-hsp.h> |
18 | |
19 | #include "mailbox.h" |
20 | |
21 | #define HSP_INT_IE(x) (0x100 + ((x) * 4)) |
22 | #define HSP_INT_IV 0x300 |
23 | #define HSP_INT_IR 0x304 |
24 | |
25 | #define HSP_INT_EMPTY_SHIFT 0 |
26 | #define HSP_INT_EMPTY_MASK 0xff |
27 | #define HSP_INT_FULL_SHIFT 8 |
28 | #define HSP_INT_FULL_MASK 0xff |
29 | |
30 | #define HSP_INT_DIMENSIONING 0x380 |
31 | #define HSP_nSM_SHIFT 0 |
32 | #define HSP_nSS_SHIFT 4 |
33 | #define HSP_nAS_SHIFT 8 |
34 | #define HSP_nDB_SHIFT 12 |
35 | #define HSP_nSI_SHIFT 16 |
36 | #define HSP_nINT_MASK 0xf |
37 | |
38 | #define HSP_DB_TRIGGER 0x0 |
39 | #define HSP_DB_ENABLE 0x4 |
40 | #define HSP_DB_RAW 0x8 |
41 | #define HSP_DB_PENDING 0xc |
42 | |
43 | #define HSP_SM_SHRD_MBOX 0x0 |
44 | #define HSP_SM_SHRD_MBOX_FULL BIT(31) |
45 | #define HSP_SM_SHRD_MBOX_FULL_INT_IE 0x04 |
46 | #define HSP_SM_SHRD_MBOX_EMPTY_INT_IE 0x08 |
47 | |
48 | #define HSP_SHRD_MBOX_TYPE1_TAG 0x40 |
49 | #define HSP_SHRD_MBOX_TYPE1_DATA0 0x48 |
50 | #define HSP_SHRD_MBOX_TYPE1_DATA1 0x4c |
51 | #define HSP_SHRD_MBOX_TYPE1_DATA2 0x50 |
52 | #define HSP_SHRD_MBOX_TYPE1_DATA3 0x54 |
53 | |
54 | #define HSP_DB_CCPLEX 1 |
55 | #define HSP_DB_BPMP 3 |
56 | #define HSP_DB_MAX 7 |
57 | |
58 | #define HSP_MBOX_TYPE_MASK 0xff |
59 | |
60 | struct tegra_hsp_channel; |
61 | struct tegra_hsp; |
62 | |
63 | struct tegra_hsp_channel { |
64 | struct tegra_hsp *hsp; |
65 | struct mbox_chan *chan; |
66 | void __iomem *regs; |
67 | }; |
68 | |
69 | struct tegra_hsp_doorbell { |
70 | struct tegra_hsp_channel channel; |
71 | struct list_head list; |
72 | const char *name; |
73 | unsigned int master; |
74 | unsigned int index; |
75 | }; |
76 | |
77 | struct tegra_hsp_sm_ops { |
78 | void (*send)(struct tegra_hsp_channel *channel, void *data); |
79 | void (*recv)(struct tegra_hsp_channel *channel); |
80 | }; |
81 | |
82 | struct tegra_hsp_mailbox { |
83 | struct tegra_hsp_channel channel; |
84 | const struct tegra_hsp_sm_ops *ops; |
85 | unsigned int index; |
86 | bool producer; |
87 | }; |
88 | |
89 | struct tegra_hsp_db_map { |
90 | const char *name; |
91 | unsigned int master; |
92 | unsigned int index; |
93 | }; |
94 | |
95 | struct tegra_hsp_soc { |
96 | const struct tegra_hsp_db_map *map; |
97 | bool has_per_mb_ie; |
98 | bool has_128_bit_mb; |
99 | unsigned int reg_stride; |
100 | }; |
101 | |
102 | struct tegra_hsp { |
103 | struct device *dev; |
104 | const struct tegra_hsp_soc *soc; |
105 | struct mbox_controller mbox_db; |
106 | struct mbox_controller mbox_sm; |
107 | void __iomem *regs; |
108 | unsigned int doorbell_irq; |
109 | unsigned int *shared_irqs; |
110 | unsigned int shared_irq; |
111 | unsigned int num_sm; |
112 | unsigned int num_as; |
113 | unsigned int num_ss; |
114 | unsigned int num_db; |
115 | unsigned int num_si; |
116 | |
117 | spinlock_t lock; |
118 | struct lock_class_key lock_key; |
119 | |
120 | struct list_head doorbells; |
121 | struct tegra_hsp_mailbox *mailboxes; |
122 | |
123 | unsigned long mask; |
124 | }; |
125 | |
126 | static inline u32 tegra_hsp_readl(struct tegra_hsp *hsp, unsigned int offset) |
127 | { |
128 | return readl(addr: hsp->regs + offset); |
129 | } |
130 | |
131 | static inline void tegra_hsp_writel(struct tegra_hsp *hsp, u32 value, |
132 | unsigned int offset) |
133 | { |
134 | writel(val: value, addr: hsp->regs + offset); |
135 | } |
136 | |
137 | static inline u32 tegra_hsp_channel_readl(struct tegra_hsp_channel *channel, |
138 | unsigned int offset) |
139 | { |
140 | return readl(addr: channel->regs + offset); |
141 | } |
142 | |
143 | static inline void tegra_hsp_channel_writel(struct tegra_hsp_channel *channel, |
144 | u32 value, unsigned int offset) |
145 | { |
146 | writel(val: value, addr: channel->regs + offset); |
147 | } |
148 | |
149 | static bool tegra_hsp_doorbell_can_ring(struct tegra_hsp_doorbell *db) |
150 | { |
151 | u32 value; |
152 | |
153 | value = tegra_hsp_channel_readl(channel: &db->channel, HSP_DB_ENABLE); |
154 | |
155 | return (value & BIT(TEGRA_HSP_DB_MASTER_CCPLEX)) != 0; |
156 | } |
157 | |
158 | static struct tegra_hsp_doorbell * |
159 | __tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master) |
160 | { |
161 | struct tegra_hsp_doorbell *entry; |
162 | |
163 | list_for_each_entry(entry, &hsp->doorbells, list) |
164 | if (entry->master == master) |
165 | return entry; |
166 | |
167 | return NULL; |
168 | } |
169 | |
170 | static struct tegra_hsp_doorbell * |
171 | tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master) |
172 | { |
173 | struct tegra_hsp_doorbell *db; |
174 | unsigned long flags; |
175 | |
176 | spin_lock_irqsave(&hsp->lock, flags); |
177 | db = __tegra_hsp_doorbell_get(hsp, master); |
178 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
179 | |
180 | return db; |
181 | } |
182 | |
183 | static irqreturn_t tegra_hsp_doorbell_irq(int irq, void *data) |
184 | { |
185 | struct tegra_hsp *hsp = data; |
186 | struct tegra_hsp_doorbell *db; |
187 | unsigned long master, value; |
188 | |
189 | db = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX); |
190 | if (!db) |
191 | return IRQ_NONE; |
192 | |
193 | value = tegra_hsp_channel_readl(channel: &db->channel, HSP_DB_PENDING); |
194 | tegra_hsp_channel_writel(channel: &db->channel, value, HSP_DB_PENDING); |
195 | |
196 | spin_lock(lock: &hsp->lock); |
197 | |
198 | for_each_set_bit(master, &value, hsp->mbox_db.num_chans) { |
199 | struct tegra_hsp_doorbell *db; |
200 | |
201 | db = __tegra_hsp_doorbell_get(hsp, master); |
202 | /* |
203 | * Depending on the bootloader chain, the CCPLEX doorbell will |
204 | * have some doorbells enabled, which means that requesting an |
205 | * interrupt will immediately fire. |
206 | * |
207 | * In that case, db->channel.chan will still be NULL here and |
208 | * cause a crash if not properly guarded. |
209 | * |
210 | * It remains to be seen if ignoring the doorbell in that case |
211 | * is the correct solution. |
212 | */ |
213 | if (db && db->channel.chan) |
214 | mbox_chan_received_data(chan: db->channel.chan, NULL); |
215 | } |
216 | |
217 | spin_unlock(lock: &hsp->lock); |
218 | |
219 | return IRQ_HANDLED; |
220 | } |
221 | |
222 | static irqreturn_t tegra_hsp_shared_irq(int irq, void *data) |
223 | { |
224 | struct tegra_hsp *hsp = data; |
225 | unsigned long bit, mask; |
226 | u32 status; |
227 | |
228 | status = tegra_hsp_readl(hsp, HSP_INT_IR) & hsp->mask; |
229 | |
230 | /* process EMPTY interrupts first */ |
231 | mask = (status >> HSP_INT_EMPTY_SHIFT) & HSP_INT_EMPTY_MASK; |
232 | |
233 | for_each_set_bit(bit, &mask, hsp->num_sm) { |
234 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; |
235 | |
236 | if (mb->producer) { |
237 | /* |
238 | * Disable EMPTY interrupts until data is sent with |
239 | * the next message. These interrupts are level- |
240 | * triggered, so if we kept them enabled they would |
241 | * constantly trigger until we next write data into |
242 | * the message. |
243 | */ |
244 | spin_lock(lock: &hsp->lock); |
245 | |
246 | hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
247 | tegra_hsp_writel(hsp, value: hsp->mask, |
248 | HSP_INT_IE(hsp->shared_irq)); |
249 | |
250 | spin_unlock(lock: &hsp->lock); |
251 | |
252 | mbox_chan_txdone(chan: mb->channel.chan, r: 0); |
253 | } |
254 | } |
255 | |
256 | /* process FULL interrupts */ |
257 | mask = (status >> HSP_INT_FULL_SHIFT) & HSP_INT_FULL_MASK; |
258 | |
259 | for_each_set_bit(bit, &mask, hsp->num_sm) { |
260 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; |
261 | |
262 | if (!mb->producer) |
263 | mb->ops->recv(&mb->channel); |
264 | } |
265 | |
266 | return IRQ_HANDLED; |
267 | } |
268 | |
269 | static struct tegra_hsp_channel * |
270 | tegra_hsp_doorbell_create(struct tegra_hsp *hsp, const char *name, |
271 | unsigned int master, unsigned int index) |
272 | { |
273 | struct tegra_hsp_doorbell *db; |
274 | unsigned int offset; |
275 | unsigned long flags; |
276 | |
277 | db = devm_kzalloc(dev: hsp->dev, size: sizeof(*db), GFP_KERNEL); |
278 | if (!db) |
279 | return ERR_PTR(error: -ENOMEM); |
280 | |
281 | offset = (1 + (hsp->num_sm / 2) + hsp->num_ss + hsp->num_as) * SZ_64K; |
282 | offset += index * hsp->soc->reg_stride; |
283 | |
284 | db->channel.regs = hsp->regs + offset; |
285 | db->channel.hsp = hsp; |
286 | |
287 | db->name = devm_kstrdup_const(dev: hsp->dev, s: name, GFP_KERNEL); |
288 | db->master = master; |
289 | db->index = index; |
290 | |
291 | spin_lock_irqsave(&hsp->lock, flags); |
292 | list_add_tail(new: &db->list, head: &hsp->doorbells); |
293 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
294 | |
295 | return &db->channel; |
296 | } |
297 | |
298 | static int tegra_hsp_doorbell_send_data(struct mbox_chan *chan, void *data) |
299 | { |
300 | struct tegra_hsp_doorbell *db = chan->con_priv; |
301 | |
302 | tegra_hsp_channel_writel(channel: &db->channel, value: 1, HSP_DB_TRIGGER); |
303 | |
304 | return 0; |
305 | } |
306 | |
307 | static int tegra_hsp_doorbell_startup(struct mbox_chan *chan) |
308 | { |
309 | struct tegra_hsp_doorbell *db = chan->con_priv; |
310 | struct tegra_hsp *hsp = db->channel.hsp; |
311 | struct tegra_hsp_doorbell *ccplex; |
312 | unsigned long flags; |
313 | u32 value; |
314 | |
315 | if (db->master >= chan->mbox->num_chans) { |
316 | dev_err(chan->mbox->dev, |
317 | "invalid master ID %u for HSP channel\n" , |
318 | db->master); |
319 | return -EINVAL; |
320 | } |
321 | |
322 | ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX); |
323 | if (!ccplex) |
324 | return -ENODEV; |
325 | |
326 | /* |
327 | * On simulation platforms the BPMP hasn't had a chance yet to mark |
328 | * the doorbell as ringable by the CCPLEX, so we want to skip extra |
329 | * checks here. |
330 | */ |
331 | if (tegra_is_silicon() && !tegra_hsp_doorbell_can_ring(db)) |
332 | return -ENODEV; |
333 | |
334 | spin_lock_irqsave(&hsp->lock, flags); |
335 | |
336 | value = tegra_hsp_channel_readl(channel: &ccplex->channel, HSP_DB_ENABLE); |
337 | value |= BIT(db->master); |
338 | tegra_hsp_channel_writel(channel: &ccplex->channel, value, HSP_DB_ENABLE); |
339 | |
340 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
341 | |
342 | return 0; |
343 | } |
344 | |
345 | static void tegra_hsp_doorbell_shutdown(struct mbox_chan *chan) |
346 | { |
347 | struct tegra_hsp_doorbell *db = chan->con_priv; |
348 | struct tegra_hsp *hsp = db->channel.hsp; |
349 | struct tegra_hsp_doorbell *ccplex; |
350 | unsigned long flags; |
351 | u32 value; |
352 | |
353 | ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX); |
354 | if (!ccplex) |
355 | return; |
356 | |
357 | spin_lock_irqsave(&hsp->lock, flags); |
358 | |
359 | value = tegra_hsp_channel_readl(channel: &ccplex->channel, HSP_DB_ENABLE); |
360 | value &= ~BIT(db->master); |
361 | tegra_hsp_channel_writel(channel: &ccplex->channel, value, HSP_DB_ENABLE); |
362 | |
363 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
364 | } |
365 | |
366 | static const struct mbox_chan_ops tegra_hsp_db_ops = { |
367 | .send_data = tegra_hsp_doorbell_send_data, |
368 | .startup = tegra_hsp_doorbell_startup, |
369 | .shutdown = tegra_hsp_doorbell_shutdown, |
370 | }; |
371 | |
372 | static void tegra_hsp_sm_send32(struct tegra_hsp_channel *channel, void *data) |
373 | { |
374 | u32 value; |
375 | |
376 | /* copy data and mark mailbox full */ |
377 | value = (u32)(unsigned long)data; |
378 | value |= HSP_SM_SHRD_MBOX_FULL; |
379 | |
380 | tegra_hsp_channel_writel(channel, value, HSP_SM_SHRD_MBOX); |
381 | } |
382 | |
383 | static void tegra_hsp_sm_recv32(struct tegra_hsp_channel *channel) |
384 | { |
385 | u32 value; |
386 | void *msg; |
387 | |
388 | value = tegra_hsp_channel_readl(channel, HSP_SM_SHRD_MBOX); |
389 | value &= ~HSP_SM_SHRD_MBOX_FULL; |
390 | msg = (void *)(unsigned long)value; |
391 | mbox_chan_received_data(chan: channel->chan, data: msg); |
392 | |
393 | /* |
394 | * Need to clear all bits here since some producers, such as TCU, depend |
395 | * on fields in the register getting cleared by the consumer. |
396 | * |
397 | * The mailbox API doesn't give the consumers a way of doing that |
398 | * explicitly, so we have to make sure we cover all possible cases. |
399 | */ |
400 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SM_SHRD_MBOX); |
401 | } |
402 | |
403 | static const struct tegra_hsp_sm_ops tegra_hsp_sm_32bit_ops = { |
404 | .send = tegra_hsp_sm_send32, |
405 | .recv = tegra_hsp_sm_recv32, |
406 | }; |
407 | |
408 | static void tegra_hsp_sm_send128(struct tegra_hsp_channel *channel, void *data) |
409 | { |
410 | u32 value[4]; |
411 | |
412 | memcpy(value, data, sizeof(value)); |
413 | |
414 | /* Copy data */ |
415 | tegra_hsp_channel_writel(channel, value: value[0], HSP_SHRD_MBOX_TYPE1_DATA0); |
416 | tegra_hsp_channel_writel(channel, value: value[1], HSP_SHRD_MBOX_TYPE1_DATA1); |
417 | tegra_hsp_channel_writel(channel, value: value[2], HSP_SHRD_MBOX_TYPE1_DATA2); |
418 | tegra_hsp_channel_writel(channel, value: value[3], HSP_SHRD_MBOX_TYPE1_DATA3); |
419 | |
420 | /* Update tag to mark mailbox full */ |
421 | tegra_hsp_channel_writel(channel, HSP_SM_SHRD_MBOX_FULL, |
422 | HSP_SHRD_MBOX_TYPE1_TAG); |
423 | } |
424 | |
425 | static void tegra_hsp_sm_recv128(struct tegra_hsp_channel *channel) |
426 | { |
427 | u32 value[4]; |
428 | void *msg; |
429 | |
430 | value[0] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA0); |
431 | value[1] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA1); |
432 | value[2] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA2); |
433 | value[3] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA3); |
434 | |
435 | msg = (void *)(unsigned long)value; |
436 | mbox_chan_received_data(chan: channel->chan, data: msg); |
437 | |
438 | /* |
439 | * Clear data registers and tag. |
440 | */ |
441 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA0); |
442 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA1); |
443 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA2); |
444 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA3); |
445 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_TAG); |
446 | } |
447 | |
448 | static const struct tegra_hsp_sm_ops tegra_hsp_sm_128bit_ops = { |
449 | .send = tegra_hsp_sm_send128, |
450 | .recv = tegra_hsp_sm_recv128, |
451 | }; |
452 | |
453 | static int tegra_hsp_mailbox_send_data(struct mbox_chan *chan, void *data) |
454 | { |
455 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
456 | struct tegra_hsp *hsp = mb->channel.hsp; |
457 | unsigned long flags; |
458 | |
459 | if (WARN_ON(!mb->producer)) |
460 | return -EPERM; |
461 | |
462 | mb->ops->send(&mb->channel, data); |
463 | |
464 | /* enable EMPTY interrupt for the shared mailbox */ |
465 | spin_lock_irqsave(&hsp->lock, flags); |
466 | |
467 | hsp->mask |= BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
468 | tegra_hsp_writel(hsp, value: hsp->mask, HSP_INT_IE(hsp->shared_irq)); |
469 | |
470 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
471 | |
472 | return 0; |
473 | } |
474 | |
475 | static int tegra_hsp_mailbox_flush(struct mbox_chan *chan, |
476 | unsigned long timeout) |
477 | { |
478 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
479 | struct tegra_hsp_channel *ch = &mb->channel; |
480 | u32 value; |
481 | |
482 | timeout = jiffies + msecs_to_jiffies(m: timeout); |
483 | |
484 | while (time_before(jiffies, timeout)) { |
485 | value = tegra_hsp_channel_readl(channel: ch, HSP_SM_SHRD_MBOX); |
486 | if ((value & HSP_SM_SHRD_MBOX_FULL) == 0) { |
487 | mbox_chan_txdone(chan, r: 0); |
488 | |
489 | /* Wait until channel is empty */ |
490 | if (chan->active_req != NULL) |
491 | continue; |
492 | |
493 | return 0; |
494 | } |
495 | |
496 | udelay(1); |
497 | } |
498 | |
499 | return -ETIME; |
500 | } |
501 | |
502 | static int tegra_hsp_mailbox_startup(struct mbox_chan *chan) |
503 | { |
504 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
505 | struct tegra_hsp_channel *ch = &mb->channel; |
506 | struct tegra_hsp *hsp = mb->channel.hsp; |
507 | unsigned long flags; |
508 | |
509 | chan->txdone_method = TXDONE_BY_IRQ; |
510 | |
511 | /* |
512 | * Shared mailboxes start out as consumers by default. FULL and EMPTY |
513 | * interrupts are coalesced at the same shared interrupt. |
514 | * |
515 | * Keep EMPTY interrupts disabled at startup and only enable them when |
516 | * the mailbox is actually full. This is required because the FULL and |
517 | * EMPTY interrupts are level-triggered, so keeping EMPTY interrupts |
518 | * enabled all the time would cause an interrupt storm while mailboxes |
519 | * are idle. |
520 | */ |
521 | |
522 | spin_lock_irqsave(&hsp->lock, flags); |
523 | |
524 | if (mb->producer) |
525 | hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
526 | else |
527 | hsp->mask |= BIT(HSP_INT_FULL_SHIFT + mb->index); |
528 | |
529 | tegra_hsp_writel(hsp, value: hsp->mask, HSP_INT_IE(hsp->shared_irq)); |
530 | |
531 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
532 | |
533 | if (hsp->soc->has_per_mb_ie) { |
534 | if (mb->producer) |
535 | tegra_hsp_channel_writel(channel: ch, value: 0x0, |
536 | HSP_SM_SHRD_MBOX_EMPTY_INT_IE); |
537 | else |
538 | tegra_hsp_channel_writel(channel: ch, value: 0x1, |
539 | HSP_SM_SHRD_MBOX_FULL_INT_IE); |
540 | } |
541 | |
542 | return 0; |
543 | } |
544 | |
545 | static void tegra_hsp_mailbox_shutdown(struct mbox_chan *chan) |
546 | { |
547 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
548 | struct tegra_hsp_channel *ch = &mb->channel; |
549 | struct tegra_hsp *hsp = mb->channel.hsp; |
550 | unsigned long flags; |
551 | |
552 | if (hsp->soc->has_per_mb_ie) { |
553 | if (mb->producer) |
554 | tegra_hsp_channel_writel(channel: ch, value: 0x0, |
555 | HSP_SM_SHRD_MBOX_EMPTY_INT_IE); |
556 | else |
557 | tegra_hsp_channel_writel(channel: ch, value: 0x0, |
558 | HSP_SM_SHRD_MBOX_FULL_INT_IE); |
559 | } |
560 | |
561 | spin_lock_irqsave(&hsp->lock, flags); |
562 | |
563 | if (mb->producer) |
564 | hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
565 | else |
566 | hsp->mask &= ~BIT(HSP_INT_FULL_SHIFT + mb->index); |
567 | |
568 | tegra_hsp_writel(hsp, value: hsp->mask, HSP_INT_IE(hsp->shared_irq)); |
569 | |
570 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
571 | } |
572 | |
573 | static const struct mbox_chan_ops tegra_hsp_sm_ops = { |
574 | .send_data = tegra_hsp_mailbox_send_data, |
575 | .flush = tegra_hsp_mailbox_flush, |
576 | .startup = tegra_hsp_mailbox_startup, |
577 | .shutdown = tegra_hsp_mailbox_shutdown, |
578 | }; |
579 | |
580 | static struct mbox_chan *tegra_hsp_db_xlate(struct mbox_controller *mbox, |
581 | const struct of_phandle_args *args) |
582 | { |
583 | struct tegra_hsp *hsp = container_of(mbox, struct tegra_hsp, mbox_db); |
584 | unsigned int type = args->args[0], master = args->args[1]; |
585 | struct tegra_hsp_channel *channel = ERR_PTR(error: -ENODEV); |
586 | struct tegra_hsp_doorbell *db; |
587 | struct mbox_chan *chan; |
588 | unsigned long flags; |
589 | unsigned int i; |
590 | |
591 | if (type != TEGRA_HSP_MBOX_TYPE_DB || !hsp->doorbell_irq) |
592 | return ERR_PTR(error: -ENODEV); |
593 | |
594 | db = tegra_hsp_doorbell_get(hsp, master); |
595 | if (db) |
596 | channel = &db->channel; |
597 | |
598 | if (IS_ERR(ptr: channel)) |
599 | return ERR_CAST(ptr: channel); |
600 | |
601 | spin_lock_irqsave(&hsp->lock, flags); |
602 | |
603 | for (i = 0; i < mbox->num_chans; i++) { |
604 | chan = &mbox->chans[i]; |
605 | if (!chan->con_priv) { |
606 | channel->chan = chan; |
607 | chan->con_priv = db; |
608 | break; |
609 | } |
610 | |
611 | chan = NULL; |
612 | } |
613 | |
614 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
615 | |
616 | return chan ?: ERR_PTR(error: -EBUSY); |
617 | } |
618 | |
619 | static struct mbox_chan *tegra_hsp_sm_xlate(struct mbox_controller *mbox, |
620 | const struct of_phandle_args *args) |
621 | { |
622 | struct tegra_hsp *hsp = container_of(mbox, struct tegra_hsp, mbox_sm); |
623 | unsigned int type = args->args[0], index; |
624 | struct tegra_hsp_mailbox *mb; |
625 | |
626 | index = args->args[1] & TEGRA_HSP_SM_MASK; |
627 | |
628 | if ((type & HSP_MBOX_TYPE_MASK) != TEGRA_HSP_MBOX_TYPE_SM || |
629 | !hsp->shared_irqs || index >= hsp->num_sm) |
630 | return ERR_PTR(error: -ENODEV); |
631 | |
632 | mb = &hsp->mailboxes[index]; |
633 | |
634 | if (type & TEGRA_HSP_MBOX_TYPE_SM_128BIT) { |
635 | if (!hsp->soc->has_128_bit_mb) |
636 | return ERR_PTR(error: -ENODEV); |
637 | |
638 | mb->ops = &tegra_hsp_sm_128bit_ops; |
639 | } else { |
640 | mb->ops = &tegra_hsp_sm_32bit_ops; |
641 | } |
642 | |
643 | if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0) |
644 | mb->producer = false; |
645 | else |
646 | mb->producer = true; |
647 | |
648 | return mb->channel.chan; |
649 | } |
650 | |
651 | static int tegra_hsp_add_doorbells(struct tegra_hsp *hsp) |
652 | { |
653 | const struct tegra_hsp_db_map *map = hsp->soc->map; |
654 | struct tegra_hsp_channel *channel; |
655 | |
656 | while (map->name) { |
657 | channel = tegra_hsp_doorbell_create(hsp, name: map->name, |
658 | master: map->master, index: map->index); |
659 | if (IS_ERR(ptr: channel)) |
660 | return PTR_ERR(ptr: channel); |
661 | |
662 | map++; |
663 | } |
664 | |
665 | return 0; |
666 | } |
667 | |
668 | static int tegra_hsp_add_mailboxes(struct tegra_hsp *hsp, struct device *dev) |
669 | { |
670 | int i; |
671 | |
672 | hsp->mailboxes = devm_kcalloc(dev, n: hsp->num_sm, size: sizeof(*hsp->mailboxes), |
673 | GFP_KERNEL); |
674 | if (!hsp->mailboxes) |
675 | return -ENOMEM; |
676 | |
677 | for (i = 0; i < hsp->num_sm; i++) { |
678 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i]; |
679 | |
680 | mb->index = i; |
681 | |
682 | mb->channel.hsp = hsp; |
683 | mb->channel.regs = hsp->regs + SZ_64K + i * SZ_32K; |
684 | mb->channel.chan = &hsp->mbox_sm.chans[i]; |
685 | mb->channel.chan->con_priv = mb; |
686 | } |
687 | |
688 | return 0; |
689 | } |
690 | |
691 | static int tegra_hsp_request_shared_irq(struct tegra_hsp *hsp) |
692 | { |
693 | unsigned int i, irq = 0; |
694 | int err; |
695 | |
696 | for (i = 0; i < hsp->num_si; i++) { |
697 | irq = hsp->shared_irqs[i]; |
698 | if (irq <= 0) |
699 | continue; |
700 | |
701 | err = devm_request_irq(dev: hsp->dev, irq, handler: tegra_hsp_shared_irq, irqflags: 0, |
702 | devname: dev_name(dev: hsp->dev), dev_id: hsp); |
703 | if (err < 0) { |
704 | dev_err(hsp->dev, "failed to request interrupt: %d\n" , |
705 | err); |
706 | continue; |
707 | } |
708 | |
709 | hsp->shared_irq = i; |
710 | |
711 | /* disable all interrupts */ |
712 | tegra_hsp_writel(hsp, value: 0, HSP_INT_IE(hsp->shared_irq)); |
713 | |
714 | dev_dbg(hsp->dev, "interrupt requested: %u\n" , irq); |
715 | |
716 | break; |
717 | } |
718 | |
719 | if (i == hsp->num_si) { |
720 | dev_err(hsp->dev, "failed to find available interrupt\n" ); |
721 | return -ENOENT; |
722 | } |
723 | |
724 | return 0; |
725 | } |
726 | |
727 | static int tegra_hsp_probe(struct platform_device *pdev) |
728 | { |
729 | struct tegra_hsp *hsp; |
730 | unsigned int i; |
731 | u32 value; |
732 | int err; |
733 | |
734 | hsp = devm_kzalloc(dev: &pdev->dev, size: sizeof(*hsp), GFP_KERNEL); |
735 | if (!hsp) |
736 | return -ENOMEM; |
737 | |
738 | hsp->dev = &pdev->dev; |
739 | hsp->soc = of_device_get_match_data(dev: &pdev->dev); |
740 | INIT_LIST_HEAD(list: &hsp->doorbells); |
741 | spin_lock_init(&hsp->lock); |
742 | |
743 | hsp->regs = devm_platform_ioremap_resource(pdev, index: 0); |
744 | if (IS_ERR(ptr: hsp->regs)) |
745 | return PTR_ERR(ptr: hsp->regs); |
746 | |
747 | value = tegra_hsp_readl(hsp, HSP_INT_DIMENSIONING); |
748 | hsp->num_sm = (value >> HSP_nSM_SHIFT) & HSP_nINT_MASK; |
749 | hsp->num_ss = (value >> HSP_nSS_SHIFT) & HSP_nINT_MASK; |
750 | hsp->num_as = (value >> HSP_nAS_SHIFT) & HSP_nINT_MASK; |
751 | hsp->num_db = (value >> HSP_nDB_SHIFT) & HSP_nINT_MASK; |
752 | hsp->num_si = (value >> HSP_nSI_SHIFT) & HSP_nINT_MASK; |
753 | |
754 | err = platform_get_irq_byname_optional(dev: pdev, name: "doorbell" ); |
755 | if (err >= 0) |
756 | hsp->doorbell_irq = err; |
757 | |
758 | if (hsp->num_si > 0) { |
759 | unsigned int count = 0; |
760 | |
761 | hsp->shared_irqs = devm_kcalloc(dev: &pdev->dev, n: hsp->num_si, |
762 | size: sizeof(*hsp->shared_irqs), |
763 | GFP_KERNEL); |
764 | if (!hsp->shared_irqs) |
765 | return -ENOMEM; |
766 | |
767 | for (i = 0; i < hsp->num_si; i++) { |
768 | char *name; |
769 | |
770 | name = kasprintf(GFP_KERNEL, fmt: "shared%u" , i); |
771 | if (!name) |
772 | return -ENOMEM; |
773 | |
774 | err = platform_get_irq_byname_optional(dev: pdev, name); |
775 | if (err >= 0) { |
776 | hsp->shared_irqs[i] = err; |
777 | count++; |
778 | } |
779 | |
780 | kfree(objp: name); |
781 | } |
782 | |
783 | if (count == 0) { |
784 | devm_kfree(dev: &pdev->dev, p: hsp->shared_irqs); |
785 | hsp->shared_irqs = NULL; |
786 | } |
787 | } |
788 | |
789 | /* setup the doorbell controller */ |
790 | hsp->mbox_db.of_xlate = tegra_hsp_db_xlate; |
791 | hsp->mbox_db.num_chans = 32; |
792 | hsp->mbox_db.dev = &pdev->dev; |
793 | hsp->mbox_db.ops = &tegra_hsp_db_ops; |
794 | |
795 | hsp->mbox_db.chans = devm_kcalloc(dev: &pdev->dev, n: hsp->mbox_db.num_chans, |
796 | size: sizeof(*hsp->mbox_db.chans), |
797 | GFP_KERNEL); |
798 | if (!hsp->mbox_db.chans) |
799 | return -ENOMEM; |
800 | |
801 | if (hsp->doorbell_irq) { |
802 | err = tegra_hsp_add_doorbells(hsp); |
803 | if (err < 0) { |
804 | dev_err(&pdev->dev, "failed to add doorbells: %d\n" , |
805 | err); |
806 | return err; |
807 | } |
808 | } |
809 | |
810 | err = devm_mbox_controller_register(dev: &pdev->dev, mbox: &hsp->mbox_db); |
811 | if (err < 0) { |
812 | dev_err(&pdev->dev, "failed to register doorbell mailbox: %d\n" , |
813 | err); |
814 | return err; |
815 | } |
816 | |
817 | /* setup the shared mailbox controller */ |
818 | hsp->mbox_sm.of_xlate = tegra_hsp_sm_xlate; |
819 | hsp->mbox_sm.num_chans = hsp->num_sm; |
820 | hsp->mbox_sm.dev = &pdev->dev; |
821 | hsp->mbox_sm.ops = &tegra_hsp_sm_ops; |
822 | |
823 | hsp->mbox_sm.chans = devm_kcalloc(dev: &pdev->dev, n: hsp->mbox_sm.num_chans, |
824 | size: sizeof(*hsp->mbox_sm.chans), |
825 | GFP_KERNEL); |
826 | if (!hsp->mbox_sm.chans) |
827 | return -ENOMEM; |
828 | |
829 | if (hsp->shared_irqs) { |
830 | err = tegra_hsp_add_mailboxes(hsp, dev: &pdev->dev); |
831 | if (err < 0) { |
832 | dev_err(&pdev->dev, "failed to add mailboxes: %d\n" , |
833 | err); |
834 | return err; |
835 | } |
836 | } |
837 | |
838 | err = devm_mbox_controller_register(dev: &pdev->dev, mbox: &hsp->mbox_sm); |
839 | if (err < 0) { |
840 | dev_err(&pdev->dev, "failed to register shared mailbox: %d\n" , |
841 | err); |
842 | return err; |
843 | } |
844 | |
845 | platform_set_drvdata(pdev, data: hsp); |
846 | |
847 | if (hsp->doorbell_irq) { |
848 | err = devm_request_irq(dev: &pdev->dev, irq: hsp->doorbell_irq, |
849 | handler: tegra_hsp_doorbell_irq, IRQF_NO_SUSPEND, |
850 | devname: dev_name(dev: &pdev->dev), dev_id: hsp); |
851 | if (err < 0) { |
852 | dev_err(&pdev->dev, |
853 | "failed to request doorbell IRQ#%u: %d\n" , |
854 | hsp->doorbell_irq, err); |
855 | return err; |
856 | } |
857 | } |
858 | |
859 | if (hsp->shared_irqs) { |
860 | err = tegra_hsp_request_shared_irq(hsp); |
861 | if (err < 0) |
862 | return err; |
863 | } |
864 | |
865 | lockdep_register_key(key: &hsp->lock_key); |
866 | lockdep_set_class(&hsp->lock, &hsp->lock_key); |
867 | |
868 | return 0; |
869 | } |
870 | |
871 | static void tegra_hsp_remove(struct platform_device *pdev) |
872 | { |
873 | struct tegra_hsp *hsp = platform_get_drvdata(pdev); |
874 | |
875 | lockdep_unregister_key(key: &hsp->lock_key); |
876 | } |
877 | |
878 | static int __maybe_unused tegra_hsp_resume(struct device *dev) |
879 | { |
880 | struct tegra_hsp *hsp = dev_get_drvdata(dev); |
881 | unsigned int i; |
882 | struct tegra_hsp_doorbell *db; |
883 | |
884 | list_for_each_entry(db, &hsp->doorbells, list) { |
885 | if (db->channel.chan) |
886 | tegra_hsp_doorbell_startup(chan: db->channel.chan); |
887 | } |
888 | |
889 | if (hsp->mailboxes) { |
890 | for (i = 0; i < hsp->num_sm; i++) { |
891 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i]; |
892 | |
893 | if (mb->channel.chan->cl) |
894 | tegra_hsp_mailbox_startup(chan: mb->channel.chan); |
895 | } |
896 | } |
897 | |
898 | return 0; |
899 | } |
900 | |
901 | static const struct dev_pm_ops tegra_hsp_pm_ops = { |
902 | .resume_noirq = tegra_hsp_resume, |
903 | }; |
904 | |
905 | static const struct tegra_hsp_db_map tegra186_hsp_db_map[] = { |
906 | { "ccplex" , TEGRA_HSP_DB_MASTER_CCPLEX, HSP_DB_CCPLEX, }, |
907 | { "bpmp" , TEGRA_HSP_DB_MASTER_BPMP, HSP_DB_BPMP, }, |
908 | { /* sentinel */ } |
909 | }; |
910 | |
911 | static const struct tegra_hsp_soc tegra186_hsp_soc = { |
912 | .map = tegra186_hsp_db_map, |
913 | .has_per_mb_ie = false, |
914 | .has_128_bit_mb = false, |
915 | .reg_stride = 0x100, |
916 | }; |
917 | |
918 | static const struct tegra_hsp_soc tegra194_hsp_soc = { |
919 | .map = tegra186_hsp_db_map, |
920 | .has_per_mb_ie = true, |
921 | .has_128_bit_mb = false, |
922 | .reg_stride = 0x100, |
923 | }; |
924 | |
925 | static const struct tegra_hsp_soc tegra234_hsp_soc = { |
926 | .map = tegra186_hsp_db_map, |
927 | .has_per_mb_ie = false, |
928 | .has_128_bit_mb = true, |
929 | .reg_stride = 0x100, |
930 | }; |
931 | |
932 | static const struct tegra_hsp_soc tegra264_hsp_soc = { |
933 | .map = tegra186_hsp_db_map, |
934 | .has_per_mb_ie = false, |
935 | .has_128_bit_mb = true, |
936 | .reg_stride = 0x1000, |
937 | }; |
938 | |
939 | static const struct of_device_id tegra_hsp_match[] = { |
940 | { .compatible = "nvidia,tegra186-hsp" , .data = &tegra186_hsp_soc }, |
941 | { .compatible = "nvidia,tegra194-hsp" , .data = &tegra194_hsp_soc }, |
942 | { .compatible = "nvidia,tegra234-hsp" , .data = &tegra234_hsp_soc }, |
943 | { .compatible = "nvidia,tegra264-hsp" , .data = &tegra264_hsp_soc }, |
944 | { } |
945 | }; |
946 | |
947 | static struct platform_driver tegra_hsp_driver = { |
948 | .driver = { |
949 | .name = "tegra-hsp" , |
950 | .of_match_table = tegra_hsp_match, |
951 | .pm = &tegra_hsp_pm_ops, |
952 | }, |
953 | .probe = tegra_hsp_probe, |
954 | .remove_new = tegra_hsp_remove, |
955 | }; |
956 | |
957 | static int __init tegra_hsp_init(void) |
958 | { |
959 | return platform_driver_register(&tegra_hsp_driver); |
960 | } |
961 | core_initcall(tegra_hsp_init); |
962 | |