1 | // SPDX-License-Identifier: GPL-2.0-only |
---|---|
2 | /* |
3 | * Copyright (c) 2016-2025, NVIDIA CORPORATION. All rights reserved. |
4 | */ |
5 | |
6 | #include <linux/delay.h> |
7 | #include <linux/interrupt.h> |
8 | #include <linux/io.h> |
9 | #include <linux/mailbox_controller.h> |
10 | #include <linux/of.h> |
11 | #include <linux/platform_device.h> |
12 | #include <linux/pm.h> |
13 | #include <linux/slab.h> |
14 | |
15 | #include <soc/tegra/fuse.h> |
16 | |
17 | #include <dt-bindings/mailbox/tegra186-hsp.h> |
18 | |
19 | #include "mailbox.h" |
20 | |
21 | #define HSP_INT_IE(x) (0x100 + ((x) * 4)) |
22 | #define HSP_INT_IV 0x300 |
23 | #define HSP_INT_IR 0x304 |
24 | |
25 | #define HSP_INT_EMPTY_SHIFT 0 |
26 | #define HSP_INT_EMPTY_MASK 0xff |
27 | #define HSP_INT_FULL_SHIFT 8 |
28 | #define HSP_INT_FULL_MASK 0xff |
29 | |
30 | #define HSP_INT_DIMENSIONING 0x380 |
31 | |
32 | #define HSP_DB_TRIGGER 0x0 |
33 | #define HSP_DB_ENABLE 0x4 |
34 | #define HSP_DB_RAW 0x8 |
35 | #define HSP_DB_PENDING 0xc |
36 | |
37 | #define HSP_SM_SHRD_MBOX 0x0 |
38 | #define HSP_SM_SHRD_MBOX_FULL BIT(31) |
39 | #define HSP_SM_SHRD_MBOX_FULL_INT_IE 0x04 |
40 | #define HSP_SM_SHRD_MBOX_EMPTY_INT_IE 0x08 |
41 | |
42 | #define HSP_SHRD_MBOX_TYPE1_TAG 0x40 |
43 | #define HSP_SHRD_MBOX_TYPE1_DATA0 0x48 |
44 | #define HSP_SHRD_MBOX_TYPE1_DATA1 0x4c |
45 | #define HSP_SHRD_MBOX_TYPE1_DATA2 0x50 |
46 | #define HSP_SHRD_MBOX_TYPE1_DATA3 0x54 |
47 | |
48 | #define HSP_DB_CCPLEX 1 |
49 | #define HSP_DB_BPMP 3 |
50 | #define HSP_DB_MAX 7 |
51 | |
52 | #define HSP_MBOX_TYPE_MASK 0xff |
53 | |
54 | struct tegra_hsp_channel; |
55 | struct tegra_hsp; |
56 | |
57 | struct tegra_hsp_channel { |
58 | struct tegra_hsp *hsp; |
59 | struct mbox_chan *chan; |
60 | void __iomem *regs; |
61 | }; |
62 | |
63 | struct tegra_hsp_doorbell { |
64 | struct tegra_hsp_channel channel; |
65 | struct list_head list; |
66 | const char *name; |
67 | unsigned int master; |
68 | unsigned int index; |
69 | }; |
70 | |
71 | struct tegra_hsp_sm_ops { |
72 | void (*send)(struct tegra_hsp_channel *channel, void *data); |
73 | void (*recv)(struct tegra_hsp_channel *channel); |
74 | }; |
75 | |
76 | struct tegra_hsp_mailbox { |
77 | struct tegra_hsp_channel channel; |
78 | const struct tegra_hsp_sm_ops *ops; |
79 | unsigned int index; |
80 | bool producer; |
81 | }; |
82 | |
83 | struct tegra_hsp_db_map { |
84 | const char *name; |
85 | unsigned int master; |
86 | unsigned int index; |
87 | }; |
88 | |
89 | struct tegra_hsp_soc { |
90 | const struct tegra_hsp_db_map *map; |
91 | bool has_per_mb_ie; |
92 | bool has_128_bit_mb; |
93 | unsigned int reg_stride; |
94 | |
95 | /* Shifts for dimensioning register. */ |
96 | unsigned int si_shift; |
97 | unsigned int db_shift; |
98 | unsigned int as_shift; |
99 | unsigned int ss_shift; |
100 | unsigned int sm_shift; |
101 | |
102 | /* Masks for dimensioning register. */ |
103 | unsigned int si_mask; |
104 | unsigned int db_mask; |
105 | unsigned int as_mask; |
106 | unsigned int ss_mask; |
107 | unsigned int sm_mask; |
108 | }; |
109 | |
110 | struct tegra_hsp { |
111 | struct device *dev; |
112 | const struct tegra_hsp_soc *soc; |
113 | struct mbox_controller mbox_db; |
114 | struct mbox_controller mbox_sm; |
115 | void __iomem *regs; |
116 | unsigned int doorbell_irq; |
117 | unsigned int *shared_irqs; |
118 | unsigned int shared_irq; |
119 | unsigned int num_sm; |
120 | unsigned int num_as; |
121 | unsigned int num_ss; |
122 | unsigned int num_db; |
123 | unsigned int num_si; |
124 | |
125 | spinlock_t lock; |
126 | struct lock_class_key lock_key; |
127 | |
128 | struct list_head doorbells; |
129 | struct tegra_hsp_mailbox *mailboxes; |
130 | |
131 | unsigned long mask; |
132 | }; |
133 | |
134 | static inline u32 tegra_hsp_readl(struct tegra_hsp *hsp, unsigned int offset) |
135 | { |
136 | return readl(addr: hsp->regs + offset); |
137 | } |
138 | |
139 | static inline void tegra_hsp_writel(struct tegra_hsp *hsp, u32 value, |
140 | unsigned int offset) |
141 | { |
142 | writel(val: value, addr: hsp->regs + offset); |
143 | } |
144 | |
145 | static inline u32 tegra_hsp_channel_readl(struct tegra_hsp_channel *channel, |
146 | unsigned int offset) |
147 | { |
148 | return readl(addr: channel->regs + offset); |
149 | } |
150 | |
151 | static inline void tegra_hsp_channel_writel(struct tegra_hsp_channel *channel, |
152 | u32 value, unsigned int offset) |
153 | { |
154 | writel(val: value, addr: channel->regs + offset); |
155 | } |
156 | |
157 | static bool tegra_hsp_doorbell_can_ring(struct tegra_hsp_doorbell *db) |
158 | { |
159 | u32 value; |
160 | |
161 | value = tegra_hsp_channel_readl(channel: &db->channel, HSP_DB_ENABLE); |
162 | |
163 | return (value & BIT(TEGRA_HSP_DB_MASTER_CCPLEX)) != 0; |
164 | } |
165 | |
166 | static struct tegra_hsp_doorbell * |
167 | __tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master) |
168 | { |
169 | struct tegra_hsp_doorbell *entry; |
170 | |
171 | list_for_each_entry(entry, &hsp->doorbells, list) |
172 | if (entry->master == master) |
173 | return entry; |
174 | |
175 | return NULL; |
176 | } |
177 | |
178 | static struct tegra_hsp_doorbell * |
179 | tegra_hsp_doorbell_get(struct tegra_hsp *hsp, unsigned int master) |
180 | { |
181 | struct tegra_hsp_doorbell *db; |
182 | unsigned long flags; |
183 | |
184 | spin_lock_irqsave(&hsp->lock, flags); |
185 | db = __tegra_hsp_doorbell_get(hsp, master); |
186 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
187 | |
188 | return db; |
189 | } |
190 | |
191 | static irqreturn_t tegra_hsp_doorbell_irq(int irq, void *data) |
192 | { |
193 | struct tegra_hsp *hsp = data; |
194 | struct tegra_hsp_doorbell *db; |
195 | unsigned long master, value; |
196 | |
197 | db = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX); |
198 | if (!db) |
199 | return IRQ_NONE; |
200 | |
201 | value = tegra_hsp_channel_readl(channel: &db->channel, HSP_DB_PENDING); |
202 | tegra_hsp_channel_writel(channel: &db->channel, value, HSP_DB_PENDING); |
203 | |
204 | spin_lock(lock: &hsp->lock); |
205 | |
206 | for_each_set_bit(master, &value, hsp->mbox_db.num_chans) { |
207 | struct tegra_hsp_doorbell *db; |
208 | |
209 | db = __tegra_hsp_doorbell_get(hsp, master); |
210 | /* |
211 | * Depending on the bootloader chain, the CCPLEX doorbell will |
212 | * have some doorbells enabled, which means that requesting an |
213 | * interrupt will immediately fire. |
214 | * |
215 | * In that case, db->channel.chan will still be NULL here and |
216 | * cause a crash if not properly guarded. |
217 | * |
218 | * It remains to be seen if ignoring the doorbell in that case |
219 | * is the correct solution. |
220 | */ |
221 | if (db && db->channel.chan) |
222 | mbox_chan_received_data(chan: db->channel.chan, NULL); |
223 | } |
224 | |
225 | spin_unlock(lock: &hsp->lock); |
226 | |
227 | return IRQ_HANDLED; |
228 | } |
229 | |
230 | static irqreturn_t tegra_hsp_shared_irq(int irq, void *data) |
231 | { |
232 | struct tegra_hsp *hsp = data; |
233 | unsigned long bit, mask; |
234 | u32 status; |
235 | |
236 | status = tegra_hsp_readl(hsp, HSP_INT_IR) & hsp->mask; |
237 | |
238 | /* process EMPTY interrupts first */ |
239 | mask = (status >> HSP_INT_EMPTY_SHIFT) & HSP_INT_EMPTY_MASK; |
240 | |
241 | for_each_set_bit(bit, &mask, hsp->num_sm) { |
242 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; |
243 | |
244 | if (mb->producer) { |
245 | /* |
246 | * Disable EMPTY interrupts until data is sent with |
247 | * the next message. These interrupts are level- |
248 | * triggered, so if we kept them enabled they would |
249 | * constantly trigger until we next write data into |
250 | * the message. |
251 | */ |
252 | spin_lock(lock: &hsp->lock); |
253 | |
254 | hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
255 | tegra_hsp_writel(hsp, value: hsp->mask, |
256 | HSP_INT_IE(hsp->shared_irq)); |
257 | |
258 | spin_unlock(lock: &hsp->lock); |
259 | |
260 | mbox_chan_txdone(chan: mb->channel.chan, r: 0); |
261 | } |
262 | } |
263 | |
264 | /* process FULL interrupts */ |
265 | mask = (status >> HSP_INT_FULL_SHIFT) & HSP_INT_FULL_MASK; |
266 | |
267 | for_each_set_bit(bit, &mask, hsp->num_sm) { |
268 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; |
269 | |
270 | if (!mb->producer) |
271 | mb->ops->recv(&mb->channel); |
272 | } |
273 | |
274 | return IRQ_HANDLED; |
275 | } |
276 | |
277 | static struct tegra_hsp_channel * |
278 | tegra_hsp_doorbell_create(struct tegra_hsp *hsp, const char *name, |
279 | unsigned int master, unsigned int index) |
280 | { |
281 | struct tegra_hsp_doorbell *db; |
282 | unsigned int offset; |
283 | unsigned long flags; |
284 | |
285 | db = devm_kzalloc(dev: hsp->dev, size: sizeof(*db), GFP_KERNEL); |
286 | if (!db) |
287 | return ERR_PTR(error: -ENOMEM); |
288 | |
289 | offset = (1 + (hsp->num_sm / 2) + hsp->num_ss + hsp->num_as) * SZ_64K; |
290 | offset += index * hsp->soc->reg_stride; |
291 | |
292 | db->channel.regs = hsp->regs + offset; |
293 | db->channel.hsp = hsp; |
294 | |
295 | db->name = devm_kstrdup_const(dev: hsp->dev, s: name, GFP_KERNEL); |
296 | db->master = master; |
297 | db->index = index; |
298 | |
299 | spin_lock_irqsave(&hsp->lock, flags); |
300 | list_add_tail(new: &db->list, head: &hsp->doorbells); |
301 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
302 | |
303 | return &db->channel; |
304 | } |
305 | |
306 | static int tegra_hsp_doorbell_send_data(struct mbox_chan *chan, void *data) |
307 | { |
308 | struct tegra_hsp_doorbell *db = chan->con_priv; |
309 | |
310 | tegra_hsp_channel_writel(channel: &db->channel, value: 1, HSP_DB_TRIGGER); |
311 | |
312 | return 0; |
313 | } |
314 | |
315 | static int tegra_hsp_doorbell_startup(struct mbox_chan *chan) |
316 | { |
317 | struct tegra_hsp_doorbell *db = chan->con_priv; |
318 | struct tegra_hsp *hsp = db->channel.hsp; |
319 | struct tegra_hsp_doorbell *ccplex; |
320 | unsigned long flags; |
321 | u32 value; |
322 | |
323 | if (db->master >= chan->mbox->num_chans) { |
324 | dev_err(chan->mbox->dev, |
325 | "invalid master ID %u for HSP channel\n", |
326 | db->master); |
327 | return -EINVAL; |
328 | } |
329 | |
330 | ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX); |
331 | if (!ccplex) |
332 | return -ENODEV; |
333 | |
334 | /* |
335 | * On simulation platforms the BPMP hasn't had a chance yet to mark |
336 | * the doorbell as ringable by the CCPLEX, so we want to skip extra |
337 | * checks here. |
338 | */ |
339 | if (tegra_is_silicon() && !tegra_hsp_doorbell_can_ring(db)) |
340 | return -ENODEV; |
341 | |
342 | spin_lock_irqsave(&hsp->lock, flags); |
343 | |
344 | value = tegra_hsp_channel_readl(channel: &ccplex->channel, HSP_DB_ENABLE); |
345 | value |= BIT(db->master); |
346 | tegra_hsp_channel_writel(channel: &ccplex->channel, value, HSP_DB_ENABLE); |
347 | |
348 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
349 | |
350 | return 0; |
351 | } |
352 | |
353 | static void tegra_hsp_doorbell_shutdown(struct mbox_chan *chan) |
354 | { |
355 | struct tegra_hsp_doorbell *db = chan->con_priv; |
356 | struct tegra_hsp *hsp = db->channel.hsp; |
357 | struct tegra_hsp_doorbell *ccplex; |
358 | unsigned long flags; |
359 | u32 value; |
360 | |
361 | ccplex = tegra_hsp_doorbell_get(hsp, TEGRA_HSP_DB_MASTER_CCPLEX); |
362 | if (!ccplex) |
363 | return; |
364 | |
365 | spin_lock_irqsave(&hsp->lock, flags); |
366 | |
367 | value = tegra_hsp_channel_readl(channel: &ccplex->channel, HSP_DB_ENABLE); |
368 | value &= ~BIT(db->master); |
369 | tegra_hsp_channel_writel(channel: &ccplex->channel, value, HSP_DB_ENABLE); |
370 | |
371 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
372 | } |
373 | |
374 | static const struct mbox_chan_ops tegra_hsp_db_ops = { |
375 | .send_data = tegra_hsp_doorbell_send_data, |
376 | .startup = tegra_hsp_doorbell_startup, |
377 | .shutdown = tegra_hsp_doorbell_shutdown, |
378 | }; |
379 | |
380 | static void tegra_hsp_sm_send32(struct tegra_hsp_channel *channel, void *data) |
381 | { |
382 | u32 value; |
383 | |
384 | /* copy data and mark mailbox full */ |
385 | value = (u32)(unsigned long)data; |
386 | value |= HSP_SM_SHRD_MBOX_FULL; |
387 | |
388 | tegra_hsp_channel_writel(channel, value, HSP_SM_SHRD_MBOX); |
389 | } |
390 | |
391 | static void tegra_hsp_sm_recv32(struct tegra_hsp_channel *channel) |
392 | { |
393 | u32 value; |
394 | void *msg; |
395 | |
396 | value = tegra_hsp_channel_readl(channel, HSP_SM_SHRD_MBOX); |
397 | value &= ~HSP_SM_SHRD_MBOX_FULL; |
398 | msg = (void *)(unsigned long)value; |
399 | |
400 | /* |
401 | * Need to clear all bits here since some producers, such as TCU, depend |
402 | * on fields in the register getting cleared by the consumer. |
403 | * |
404 | * The mailbox API doesn't give the consumers a way of doing that |
405 | * explicitly, so we have to make sure we cover all possible cases. |
406 | */ |
407 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SM_SHRD_MBOX); |
408 | |
409 | mbox_chan_received_data(chan: channel->chan, data: msg); |
410 | } |
411 | |
412 | static const struct tegra_hsp_sm_ops tegra_hsp_sm_32bit_ops = { |
413 | .send = tegra_hsp_sm_send32, |
414 | .recv = tegra_hsp_sm_recv32, |
415 | }; |
416 | |
417 | static void tegra_hsp_sm_send128(struct tegra_hsp_channel *channel, void *data) |
418 | { |
419 | u32 value[4]; |
420 | |
421 | memcpy(value, data, sizeof(value)); |
422 | |
423 | /* Copy data */ |
424 | tegra_hsp_channel_writel(channel, value: value[0], HSP_SHRD_MBOX_TYPE1_DATA0); |
425 | tegra_hsp_channel_writel(channel, value: value[1], HSP_SHRD_MBOX_TYPE1_DATA1); |
426 | tegra_hsp_channel_writel(channel, value: value[2], HSP_SHRD_MBOX_TYPE1_DATA2); |
427 | tegra_hsp_channel_writel(channel, value: value[3], HSP_SHRD_MBOX_TYPE1_DATA3); |
428 | |
429 | /* Update tag to mark mailbox full */ |
430 | tegra_hsp_channel_writel(channel, HSP_SM_SHRD_MBOX_FULL, |
431 | HSP_SHRD_MBOX_TYPE1_TAG); |
432 | } |
433 | |
434 | static void tegra_hsp_sm_recv128(struct tegra_hsp_channel *channel) |
435 | { |
436 | u32 value[4]; |
437 | void *msg; |
438 | |
439 | value[0] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA0); |
440 | value[1] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA1); |
441 | value[2] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA2); |
442 | value[3] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA3); |
443 | |
444 | msg = (void *)(unsigned long)value; |
445 | |
446 | /* |
447 | * Clear data registers and tag. |
448 | */ |
449 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA0); |
450 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA1); |
451 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA2); |
452 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_DATA3); |
453 | tegra_hsp_channel_writel(channel, value: 0x0, HSP_SHRD_MBOX_TYPE1_TAG); |
454 | |
455 | mbox_chan_received_data(chan: channel->chan, data: msg); |
456 | } |
457 | |
458 | static const struct tegra_hsp_sm_ops tegra_hsp_sm_128bit_ops = { |
459 | .send = tegra_hsp_sm_send128, |
460 | .recv = tegra_hsp_sm_recv128, |
461 | }; |
462 | |
463 | static int tegra_hsp_mailbox_send_data(struct mbox_chan *chan, void *data) |
464 | { |
465 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
466 | struct tegra_hsp *hsp = mb->channel.hsp; |
467 | unsigned long flags; |
468 | |
469 | if (WARN_ON(!mb->producer)) |
470 | return -EPERM; |
471 | |
472 | mb->ops->send(&mb->channel, data); |
473 | |
474 | /* enable EMPTY interrupt for the shared mailbox */ |
475 | spin_lock_irqsave(&hsp->lock, flags); |
476 | |
477 | hsp->mask |= BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
478 | tegra_hsp_writel(hsp, value: hsp->mask, HSP_INT_IE(hsp->shared_irq)); |
479 | |
480 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
481 | |
482 | return 0; |
483 | } |
484 | |
485 | static int tegra_hsp_mailbox_flush(struct mbox_chan *chan, |
486 | unsigned long timeout) |
487 | { |
488 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
489 | struct tegra_hsp_channel *ch = &mb->channel; |
490 | u32 value; |
491 | |
492 | timeout = jiffies + msecs_to_jiffies(m: timeout); |
493 | |
494 | while (time_before(jiffies, timeout)) { |
495 | value = tegra_hsp_channel_readl(channel: ch, HSP_SM_SHRD_MBOX); |
496 | if ((value & HSP_SM_SHRD_MBOX_FULL) == 0) { |
497 | mbox_chan_txdone(chan, r: 0); |
498 | |
499 | /* Wait until channel is empty */ |
500 | if (chan->active_req != NULL) |
501 | continue; |
502 | |
503 | return 0; |
504 | } |
505 | |
506 | udelay(usec: 1); |
507 | } |
508 | |
509 | return -ETIME; |
510 | } |
511 | |
512 | static int tegra_hsp_mailbox_startup(struct mbox_chan *chan) |
513 | { |
514 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
515 | struct tegra_hsp_channel *ch = &mb->channel; |
516 | struct tegra_hsp *hsp = mb->channel.hsp; |
517 | unsigned long flags; |
518 | |
519 | chan->txdone_method = TXDONE_BY_IRQ; |
520 | |
521 | /* |
522 | * Shared mailboxes start out as consumers by default. FULL and EMPTY |
523 | * interrupts are coalesced at the same shared interrupt. |
524 | * |
525 | * Keep EMPTY interrupts disabled at startup and only enable them when |
526 | * the mailbox is actually full. This is required because the FULL and |
527 | * EMPTY interrupts are level-triggered, so keeping EMPTY interrupts |
528 | * enabled all the time would cause an interrupt storm while mailboxes |
529 | * are idle. |
530 | */ |
531 | |
532 | spin_lock_irqsave(&hsp->lock, flags); |
533 | |
534 | if (mb->producer) |
535 | hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
536 | else |
537 | hsp->mask |= BIT(HSP_INT_FULL_SHIFT + mb->index); |
538 | |
539 | tegra_hsp_writel(hsp, value: hsp->mask, HSP_INT_IE(hsp->shared_irq)); |
540 | |
541 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
542 | |
543 | if (hsp->soc->has_per_mb_ie) { |
544 | if (mb->producer) |
545 | tegra_hsp_channel_writel(channel: ch, value: 0x0, |
546 | HSP_SM_SHRD_MBOX_EMPTY_INT_IE); |
547 | else |
548 | tegra_hsp_channel_writel(channel: ch, value: 0x1, |
549 | HSP_SM_SHRD_MBOX_FULL_INT_IE); |
550 | } |
551 | |
552 | return 0; |
553 | } |
554 | |
555 | static void tegra_hsp_mailbox_shutdown(struct mbox_chan *chan) |
556 | { |
557 | struct tegra_hsp_mailbox *mb = chan->con_priv; |
558 | struct tegra_hsp_channel *ch = &mb->channel; |
559 | struct tegra_hsp *hsp = mb->channel.hsp; |
560 | unsigned long flags; |
561 | |
562 | if (hsp->soc->has_per_mb_ie) { |
563 | if (mb->producer) |
564 | tegra_hsp_channel_writel(channel: ch, value: 0x0, |
565 | HSP_SM_SHRD_MBOX_EMPTY_INT_IE); |
566 | else |
567 | tegra_hsp_channel_writel(channel: ch, value: 0x0, |
568 | HSP_SM_SHRD_MBOX_FULL_INT_IE); |
569 | } |
570 | |
571 | spin_lock_irqsave(&hsp->lock, flags); |
572 | |
573 | if (mb->producer) |
574 | hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); |
575 | else |
576 | hsp->mask &= ~BIT(HSP_INT_FULL_SHIFT + mb->index); |
577 | |
578 | tegra_hsp_writel(hsp, value: hsp->mask, HSP_INT_IE(hsp->shared_irq)); |
579 | |
580 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
581 | } |
582 | |
583 | static const struct mbox_chan_ops tegra_hsp_sm_ops = { |
584 | .send_data = tegra_hsp_mailbox_send_data, |
585 | .flush = tegra_hsp_mailbox_flush, |
586 | .startup = tegra_hsp_mailbox_startup, |
587 | .shutdown = tegra_hsp_mailbox_shutdown, |
588 | }; |
589 | |
590 | static struct mbox_chan *tegra_hsp_db_xlate(struct mbox_controller *mbox, |
591 | const struct of_phandle_args *args) |
592 | { |
593 | struct tegra_hsp *hsp = container_of(mbox, struct tegra_hsp, mbox_db); |
594 | unsigned int type = args->args[0], master = args->args[1]; |
595 | struct tegra_hsp_channel *channel = ERR_PTR(error: -ENODEV); |
596 | struct tegra_hsp_doorbell *db; |
597 | struct mbox_chan *chan; |
598 | unsigned long flags; |
599 | unsigned int i; |
600 | |
601 | if (type != TEGRA_HSP_MBOX_TYPE_DB || !hsp->doorbell_irq) |
602 | return ERR_PTR(error: -ENODEV); |
603 | |
604 | db = tegra_hsp_doorbell_get(hsp, master); |
605 | if (db) |
606 | channel = &db->channel; |
607 | |
608 | if (IS_ERR(ptr: channel)) |
609 | return ERR_CAST(ptr: channel); |
610 | |
611 | spin_lock_irqsave(&hsp->lock, flags); |
612 | |
613 | for (i = 0; i < mbox->num_chans; i++) { |
614 | chan = &mbox->chans[i]; |
615 | if (!chan->con_priv) { |
616 | channel->chan = chan; |
617 | chan->con_priv = db; |
618 | break; |
619 | } |
620 | |
621 | chan = NULL; |
622 | } |
623 | |
624 | spin_unlock_irqrestore(lock: &hsp->lock, flags); |
625 | |
626 | return chan ?: ERR_PTR(error: -EBUSY); |
627 | } |
628 | |
629 | static struct mbox_chan *tegra_hsp_sm_xlate(struct mbox_controller *mbox, |
630 | const struct of_phandle_args *args) |
631 | { |
632 | struct tegra_hsp *hsp = container_of(mbox, struct tegra_hsp, mbox_sm); |
633 | unsigned int type = args->args[0], index; |
634 | struct tegra_hsp_mailbox *mb; |
635 | |
636 | index = args->args[1] & TEGRA_HSP_SM_MASK; |
637 | |
638 | if ((type & HSP_MBOX_TYPE_MASK) != TEGRA_HSP_MBOX_TYPE_SM || |
639 | !hsp->shared_irqs || index >= hsp->num_sm) |
640 | return ERR_PTR(error: -ENODEV); |
641 | |
642 | mb = &hsp->mailboxes[index]; |
643 | |
644 | if (type & TEGRA_HSP_MBOX_TYPE_SM_128BIT) { |
645 | if (!hsp->soc->has_128_bit_mb) |
646 | return ERR_PTR(error: -ENODEV); |
647 | |
648 | mb->ops = &tegra_hsp_sm_128bit_ops; |
649 | } else { |
650 | mb->ops = &tegra_hsp_sm_32bit_ops; |
651 | } |
652 | |
653 | if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0) |
654 | mb->producer = false; |
655 | else |
656 | mb->producer = true; |
657 | |
658 | return mb->channel.chan; |
659 | } |
660 | |
661 | static int tegra_hsp_add_doorbells(struct tegra_hsp *hsp) |
662 | { |
663 | const struct tegra_hsp_db_map *map = hsp->soc->map; |
664 | struct tegra_hsp_channel *channel; |
665 | |
666 | while (map->name) { |
667 | channel = tegra_hsp_doorbell_create(hsp, name: map->name, |
668 | master: map->master, index: map->index); |
669 | if (IS_ERR(ptr: channel)) |
670 | return PTR_ERR(ptr: channel); |
671 | |
672 | map++; |
673 | } |
674 | |
675 | return 0; |
676 | } |
677 | |
678 | static int tegra_hsp_add_mailboxes(struct tegra_hsp *hsp, struct device *dev) |
679 | { |
680 | int i; |
681 | |
682 | hsp->mailboxes = devm_kcalloc(dev, n: hsp->num_sm, size: sizeof(*hsp->mailboxes), |
683 | GFP_KERNEL); |
684 | if (!hsp->mailboxes) |
685 | return -ENOMEM; |
686 | |
687 | for (i = 0; i < hsp->num_sm; i++) { |
688 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i]; |
689 | |
690 | mb->index = i; |
691 | |
692 | mb->channel.hsp = hsp; |
693 | mb->channel.regs = hsp->regs + SZ_64K + i * SZ_32K; |
694 | mb->channel.chan = &hsp->mbox_sm.chans[i]; |
695 | mb->channel.chan->con_priv = mb; |
696 | } |
697 | |
698 | return 0; |
699 | } |
700 | |
701 | static int tegra_hsp_request_shared_irq(struct tegra_hsp *hsp) |
702 | { |
703 | unsigned int i, irq = 0; |
704 | int err; |
705 | |
706 | for (i = 0; i < hsp->num_si; i++) { |
707 | irq = hsp->shared_irqs[i]; |
708 | if (irq <= 0) |
709 | continue; |
710 | |
711 | err = devm_request_irq(dev: hsp->dev, irq, handler: tegra_hsp_shared_irq, irqflags: 0, |
712 | devname: dev_name(dev: hsp->dev), dev_id: hsp); |
713 | if (err < 0) { |
714 | dev_err(hsp->dev, "failed to request interrupt: %d\n", |
715 | err); |
716 | continue; |
717 | } |
718 | |
719 | hsp->shared_irq = i; |
720 | |
721 | /* disable all interrupts */ |
722 | tegra_hsp_writel(hsp, value: 0, HSP_INT_IE(hsp->shared_irq)); |
723 | |
724 | dev_dbg(hsp->dev, "interrupt requested: %u\n", irq); |
725 | |
726 | break; |
727 | } |
728 | |
729 | if (i == hsp->num_si) { |
730 | dev_err(hsp->dev, "failed to find available interrupt\n"); |
731 | return -ENOENT; |
732 | } |
733 | |
734 | return 0; |
735 | } |
736 | |
737 | static int tegra_hsp_probe(struct platform_device *pdev) |
738 | { |
739 | struct tegra_hsp *hsp; |
740 | unsigned int i; |
741 | u32 value; |
742 | int err; |
743 | |
744 | hsp = devm_kzalloc(dev: &pdev->dev, size: sizeof(*hsp), GFP_KERNEL); |
745 | if (!hsp) |
746 | return -ENOMEM; |
747 | |
748 | hsp->dev = &pdev->dev; |
749 | hsp->soc = of_device_get_match_data(dev: &pdev->dev); |
750 | INIT_LIST_HEAD(list: &hsp->doorbells); |
751 | spin_lock_init(&hsp->lock); |
752 | |
753 | hsp->regs = devm_platform_ioremap_resource(pdev, index: 0); |
754 | if (IS_ERR(ptr: hsp->regs)) |
755 | return PTR_ERR(ptr: hsp->regs); |
756 | |
757 | value = tegra_hsp_readl(hsp, HSP_INT_DIMENSIONING); |
758 | hsp->num_sm = (value >> hsp->soc->sm_shift) & hsp->soc->sm_mask; |
759 | hsp->num_ss = (value >> hsp->soc->ss_shift) & hsp->soc->ss_mask; |
760 | hsp->num_as = (value >> hsp->soc->as_shift) & hsp->soc->as_mask; |
761 | hsp->num_db = (value >> hsp->soc->db_shift) & hsp->soc->db_mask; |
762 | hsp->num_si = (value >> hsp->soc->si_shift) & hsp->soc->si_mask; |
763 | |
764 | err = platform_get_irq_byname_optional(dev: pdev, name: "doorbell"); |
765 | if (err >= 0) |
766 | hsp->doorbell_irq = err; |
767 | |
768 | if (hsp->num_si > 0) { |
769 | unsigned int count = 0; |
770 | |
771 | hsp->shared_irqs = devm_kcalloc(dev: &pdev->dev, n: hsp->num_si, |
772 | size: sizeof(*hsp->shared_irqs), |
773 | GFP_KERNEL); |
774 | if (!hsp->shared_irqs) |
775 | return -ENOMEM; |
776 | |
777 | for (i = 0; i < hsp->num_si; i++) { |
778 | char *name; |
779 | |
780 | name = kasprintf(GFP_KERNEL, fmt: "shared%u", i); |
781 | if (!name) |
782 | return -ENOMEM; |
783 | |
784 | err = platform_get_irq_byname_optional(dev: pdev, name); |
785 | if (err >= 0) { |
786 | hsp->shared_irqs[i] = err; |
787 | count++; |
788 | } |
789 | |
790 | kfree(objp: name); |
791 | } |
792 | |
793 | if (count == 0) { |
794 | devm_kfree(dev: &pdev->dev, p: hsp->shared_irqs); |
795 | hsp->shared_irqs = NULL; |
796 | } |
797 | } |
798 | |
799 | /* setup the doorbell controller */ |
800 | hsp->mbox_db.of_xlate = tegra_hsp_db_xlate; |
801 | hsp->mbox_db.num_chans = 32; |
802 | hsp->mbox_db.dev = &pdev->dev; |
803 | hsp->mbox_db.ops = &tegra_hsp_db_ops; |
804 | |
805 | hsp->mbox_db.chans = devm_kcalloc(dev: &pdev->dev, n: hsp->mbox_db.num_chans, |
806 | size: sizeof(*hsp->mbox_db.chans), |
807 | GFP_KERNEL); |
808 | if (!hsp->mbox_db.chans) |
809 | return -ENOMEM; |
810 | |
811 | if (hsp->doorbell_irq) { |
812 | err = tegra_hsp_add_doorbells(hsp); |
813 | if (err < 0) { |
814 | dev_err(&pdev->dev, "failed to add doorbells: %d\n", |
815 | err); |
816 | return err; |
817 | } |
818 | } |
819 | |
820 | err = devm_mbox_controller_register(dev: &pdev->dev, mbox: &hsp->mbox_db); |
821 | if (err < 0) { |
822 | dev_err(&pdev->dev, "failed to register doorbell mailbox: %d\n", |
823 | err); |
824 | return err; |
825 | } |
826 | |
827 | /* setup the shared mailbox controller */ |
828 | hsp->mbox_sm.of_xlate = tegra_hsp_sm_xlate; |
829 | hsp->mbox_sm.num_chans = hsp->num_sm; |
830 | hsp->mbox_sm.dev = &pdev->dev; |
831 | hsp->mbox_sm.ops = &tegra_hsp_sm_ops; |
832 | |
833 | hsp->mbox_sm.chans = devm_kcalloc(dev: &pdev->dev, n: hsp->mbox_sm.num_chans, |
834 | size: sizeof(*hsp->mbox_sm.chans), |
835 | GFP_KERNEL); |
836 | if (!hsp->mbox_sm.chans) |
837 | return -ENOMEM; |
838 | |
839 | if (hsp->shared_irqs) { |
840 | err = tegra_hsp_add_mailboxes(hsp, dev: &pdev->dev); |
841 | if (err < 0) { |
842 | dev_err(&pdev->dev, "failed to add mailboxes: %d\n", |
843 | err); |
844 | return err; |
845 | } |
846 | } |
847 | |
848 | err = devm_mbox_controller_register(dev: &pdev->dev, mbox: &hsp->mbox_sm); |
849 | if (err < 0) { |
850 | dev_err(&pdev->dev, "failed to register shared mailbox: %d\n", |
851 | err); |
852 | return err; |
853 | } |
854 | |
855 | platform_set_drvdata(pdev, data: hsp); |
856 | |
857 | if (hsp->doorbell_irq) { |
858 | err = devm_request_irq(dev: &pdev->dev, irq: hsp->doorbell_irq, |
859 | handler: tegra_hsp_doorbell_irq, IRQF_NO_SUSPEND, |
860 | devname: dev_name(dev: &pdev->dev), dev_id: hsp); |
861 | if (err < 0) { |
862 | dev_err(&pdev->dev, |
863 | "failed to request doorbell IRQ#%u: %d\n", |
864 | hsp->doorbell_irq, err); |
865 | return err; |
866 | } |
867 | } |
868 | |
869 | if (hsp->shared_irqs) { |
870 | err = tegra_hsp_request_shared_irq(hsp); |
871 | if (err < 0) |
872 | return err; |
873 | } |
874 | |
875 | lockdep_register_key(key: &hsp->lock_key); |
876 | lockdep_set_class(&hsp->lock, &hsp->lock_key); |
877 | |
878 | return 0; |
879 | } |
880 | |
881 | static void tegra_hsp_remove(struct platform_device *pdev) |
882 | { |
883 | struct tegra_hsp *hsp = platform_get_drvdata(pdev); |
884 | |
885 | lockdep_unregister_key(key: &hsp->lock_key); |
886 | } |
887 | |
888 | static int __maybe_unused tegra_hsp_resume(struct device *dev) |
889 | { |
890 | struct tegra_hsp *hsp = dev_get_drvdata(dev); |
891 | unsigned int i; |
892 | struct tegra_hsp_doorbell *db; |
893 | |
894 | list_for_each_entry(db, &hsp->doorbells, list) { |
895 | if (db->channel.chan) |
896 | tegra_hsp_doorbell_startup(chan: db->channel.chan); |
897 | } |
898 | |
899 | if (hsp->mailboxes) { |
900 | for (i = 0; i < hsp->num_sm; i++) { |
901 | struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i]; |
902 | |
903 | if (mb->channel.chan->cl) |
904 | tegra_hsp_mailbox_startup(chan: mb->channel.chan); |
905 | } |
906 | } |
907 | |
908 | return 0; |
909 | } |
910 | |
911 | static const struct dev_pm_ops tegra_hsp_pm_ops = { |
912 | .resume_noirq = tegra_hsp_resume, |
913 | }; |
914 | |
915 | static const struct tegra_hsp_db_map tegra186_hsp_db_map[] = { |
916 | { "ccplex", TEGRA_HSP_DB_MASTER_CCPLEX, HSP_DB_CCPLEX, }, |
917 | { "bpmp", TEGRA_HSP_DB_MASTER_BPMP, HSP_DB_BPMP, }, |
918 | { /* sentinel */ } |
919 | }; |
920 | |
921 | static const struct tegra_hsp_soc tegra186_hsp_soc = { |
922 | .map = tegra186_hsp_db_map, |
923 | .has_per_mb_ie = false, |
924 | .has_128_bit_mb = false, |
925 | .reg_stride = 0x100, |
926 | .si_shift = 16, |
927 | .db_shift = 12, |
928 | .as_shift = 8, |
929 | .ss_shift = 4, |
930 | .sm_shift = 0, |
931 | .si_mask = 0xf, |
932 | .db_mask = 0xf, |
933 | .as_mask = 0xf, |
934 | .ss_mask = 0xf, |
935 | .sm_mask = 0xf, |
936 | }; |
937 | |
938 | static const struct tegra_hsp_soc tegra194_hsp_soc = { |
939 | .map = tegra186_hsp_db_map, |
940 | .has_per_mb_ie = true, |
941 | .has_128_bit_mb = false, |
942 | .reg_stride = 0x100, |
943 | .si_shift = 16, |
944 | .db_shift = 12, |
945 | .as_shift = 8, |
946 | .ss_shift = 4, |
947 | .sm_shift = 0, |
948 | .si_mask = 0xf, |
949 | .db_mask = 0xf, |
950 | .as_mask = 0xf, |
951 | .ss_mask = 0xf, |
952 | .sm_mask = 0xf, |
953 | }; |
954 | |
955 | static const struct tegra_hsp_soc tegra234_hsp_soc = { |
956 | .map = tegra186_hsp_db_map, |
957 | .has_per_mb_ie = false, |
958 | .has_128_bit_mb = true, |
959 | .reg_stride = 0x100, |
960 | .si_shift = 16, |
961 | .db_shift = 12, |
962 | .as_shift = 8, |
963 | .ss_shift = 4, |
964 | .sm_shift = 0, |
965 | .si_mask = 0xf, |
966 | .db_mask = 0xf, |
967 | .as_mask = 0xf, |
968 | .ss_mask = 0xf, |
969 | .sm_mask = 0xf, |
970 | }; |
971 | |
972 | static const struct tegra_hsp_soc tegra264_hsp_soc = { |
973 | .map = tegra186_hsp_db_map, |
974 | .has_per_mb_ie = false, |
975 | .has_128_bit_mb = true, |
976 | .reg_stride = 0x1000, |
977 | .si_shift = 17, |
978 | .db_shift = 12, |
979 | .as_shift = 8, |
980 | .ss_shift = 4, |
981 | .sm_shift = 0, |
982 | .si_mask = 0x1f, |
983 | .db_mask = 0x1f, |
984 | .as_mask = 0xf, |
985 | .ss_mask = 0xf, |
986 | .sm_mask = 0xf, |
987 | }; |
988 | |
989 | static const struct of_device_id tegra_hsp_match[] = { |
990 | { .compatible = "nvidia,tegra186-hsp", .data = &tegra186_hsp_soc }, |
991 | { .compatible = "nvidia,tegra194-hsp", .data = &tegra194_hsp_soc }, |
992 | { .compatible = "nvidia,tegra234-hsp", .data = &tegra234_hsp_soc }, |
993 | { .compatible = "nvidia,tegra264-hsp", .data = &tegra264_hsp_soc }, |
994 | { } |
995 | }; |
996 | |
997 | static struct platform_driver tegra_hsp_driver = { |
998 | .driver = { |
999 | .name = "tegra-hsp", |
1000 | .of_match_table = tegra_hsp_match, |
1001 | .pm = &tegra_hsp_pm_ops, |
1002 | }, |
1003 | .probe = tegra_hsp_probe, |
1004 | .remove = tegra_hsp_remove, |
1005 | }; |
1006 | |
1007 | static int __init tegra_hsp_init(void) |
1008 | { |
1009 | return platform_driver_register(&tegra_hsp_driver); |
1010 | } |
1011 | core_initcall(tegra_hsp_init); |
1012 |
Definitions
- tegra_hsp_channel
- tegra_hsp_doorbell
- tegra_hsp_sm_ops
- tegra_hsp_mailbox
- tegra_hsp_db_map
- tegra_hsp_soc
- tegra_hsp
- tegra_hsp_readl
- tegra_hsp_writel
- tegra_hsp_channel_readl
- tegra_hsp_channel_writel
- tegra_hsp_doorbell_can_ring
- __tegra_hsp_doorbell_get
- tegra_hsp_doorbell_get
- tegra_hsp_doorbell_irq
- tegra_hsp_shared_irq
- tegra_hsp_doorbell_create
- tegra_hsp_doorbell_send_data
- tegra_hsp_doorbell_startup
- tegra_hsp_doorbell_shutdown
- tegra_hsp_db_ops
- tegra_hsp_sm_send32
- tegra_hsp_sm_recv32
- tegra_hsp_sm_32bit_ops
- tegra_hsp_sm_send128
- tegra_hsp_sm_recv128
- tegra_hsp_sm_128bit_ops
- tegra_hsp_mailbox_send_data
- tegra_hsp_mailbox_flush
- tegra_hsp_mailbox_startup
- tegra_hsp_mailbox_shutdown
- tegra_hsp_sm_ops
- tegra_hsp_db_xlate
- tegra_hsp_sm_xlate
- tegra_hsp_add_doorbells
- tegra_hsp_add_mailboxes
- tegra_hsp_request_shared_irq
- tegra_hsp_probe
- tegra_hsp_remove
- tegra_hsp_resume
- tegra_hsp_pm_ops
- tegra186_hsp_db_map
- tegra186_hsp_soc
- tegra194_hsp_soc
- tegra234_hsp_soc
- tegra264_hsp_soc
- tegra_hsp_match
- tegra_hsp_driver
Improve your Profiling and Debugging skills
Find out more