1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/drm_device.h>
7#include <drm/drm_managed.h>
8#include <linux/bitfield.h>
9#include <linux/interrupt.h>
10#include <linux/iopoll.h>
11#include <linux/slab.h>
12#include <linux/xarray.h>
13
14#define CREATE_TRACE_POINTS
15#include <trace/events/amdxdna.h>
16
17#include "amdxdna_mailbox.h"
18
19#define MB_ERR(chann, fmt, args...) \
20({ \
21 typeof(chann) _chann = chann; \
22 dev_err((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \
23 (_chann)->msix_irq, ##args); \
24})
25#define MB_DBG(chann, fmt, args...) \
26({ \
27 typeof(chann) _chann = chann; \
28 dev_dbg((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \
29 (_chann)->msix_irq, ##args); \
30})
31#define MB_WARN_ONCE(chann, fmt, args...) \
32({ \
33 typeof(chann) _chann = chann; \
34 dev_warn_once((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \
35 (_chann)->msix_irq, ##args); \
36})
37
38#define MAGIC_VAL 0x1D000000U
39#define MAGIC_VAL_MASK 0xFF000000
40#define MAX_MSG_ID_ENTRIES 256
41#define MSG_RX_TIMER 200 /* milliseconds */
42#define MAILBOX_NAME "xdna_mailbox"
43
44enum channel_res_type {
45 CHAN_RES_X2I,
46 CHAN_RES_I2X,
47 CHAN_RES_NUM
48};
49
50struct mailbox {
51 struct device *dev;
52 struct xdna_mailbox_res res;
53};
54
55struct mailbox_channel {
56 struct mailbox *mb;
57 struct xdna_mailbox_chann_res res[CHAN_RES_NUM];
58 int msix_irq;
59 u32 iohub_int_addr;
60 struct xarray chan_xa;
61 u32 next_msgid;
62 u32 x2i_tail;
63
64 /* Received msg related fields */
65 struct workqueue_struct *work_q;
66 struct work_struct rx_work;
67 u32 i2x_head;
68 bool bad_state;
69};
70
71#define MSG_BODY_SZ GENMASK(10, 0)
72#define MSG_PROTO_VER GENMASK(23, 16)
73struct xdna_msg_header {
74 __u32 total_size;
75 __u32 sz_ver;
76 __u32 id;
77 __u32 opcode;
78} __packed;
79
80static_assert(sizeof(struct xdna_msg_header) == 16);
81
82struct mailbox_pkg {
83 struct xdna_msg_header header;
84 __u32 payload[];
85};
86
87/* The protocol version. */
88#define MSG_PROTOCOL_VERSION 0x1
89/* The tombstone value. */
90#define TOMBSTONE 0xDEADFACE
91
92struct mailbox_msg {
93 void *handle;
94 int (*notify_cb)(void *handle, void __iomem *data, size_t size);
95 size_t pkg_size; /* package size in bytes */
96 struct mailbox_pkg pkg;
97};
98
99static void mailbox_reg_write(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 data)
100{
101 struct xdna_mailbox_res *mb_res = &mb_chann->mb->res;
102 void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg;
103
104 writel(val: data, addr: ringbuf_addr);
105}
106
107static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg)
108{
109 struct xdna_mailbox_res *mb_res = &mb_chann->mb->res;
110 void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg;
111
112 return readl(addr: ringbuf_addr);
113}
114
115static int mailbox_reg_read_non_zero(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 *val)
116{
117 struct xdna_mailbox_res *mb_res = &mb_chann->mb->res;
118 void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg;
119 int ret, value;
120
121 /* Poll till value is not zero */
122 ret = readx_poll_timeout(readl, ringbuf_addr, value,
123 value, 1 /* us */, 100);
124 if (ret < 0)
125 return ret;
126
127 *val = value;
128 return 0;
129}
130
131static inline void
132mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val)
133{
134 mailbox_reg_write(mb_chann, mbox_reg: mb_chann->res[CHAN_RES_I2X].mb_head_ptr_reg, data: headptr_val);
135 mb_chann->i2x_head = headptr_val;
136}
137
138static inline void
139mailbox_set_tailptr(struct mailbox_channel *mb_chann, u32 tailptr_val)
140{
141 mailbox_reg_write(mb_chann, mbox_reg: mb_chann->res[CHAN_RES_X2I].mb_tail_ptr_reg, data: tailptr_val);
142 mb_chann->x2i_tail = tailptr_val;
143}
144
145static inline u32
146mailbox_get_headptr(struct mailbox_channel *mb_chann, enum channel_res_type type)
147{
148 return mailbox_reg_read(mb_chann, mbox_reg: mb_chann->res[type].mb_head_ptr_reg);
149}
150
151static inline u32
152mailbox_get_tailptr(struct mailbox_channel *mb_chann, enum channel_res_type type)
153{
154 return mailbox_reg_read(mb_chann, mbox_reg: mb_chann->res[type].mb_tail_ptr_reg);
155}
156
157static inline u32
158mailbox_get_ringbuf_size(struct mailbox_channel *mb_chann, enum channel_res_type type)
159{
160 return mb_chann->res[type].rb_size;
161}
162
163static inline int mailbox_validate_msgid(int msg_id)
164{
165 return (msg_id & MAGIC_VAL_MASK) == MAGIC_VAL;
166}
167
168static int mailbox_acquire_msgid(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg)
169{
170 u32 msg_id;
171 int ret;
172
173 ret = xa_alloc_cyclic_irq(xa: &mb_chann->chan_xa, id: &msg_id, entry: mb_msg,
174 XA_LIMIT(0, MAX_MSG_ID_ENTRIES - 1),
175 next: &mb_chann->next_msgid, GFP_NOWAIT);
176 if (ret < 0)
177 return ret;
178
179 /*
180 * Add MAGIC_VAL to the higher bits.
181 */
182 msg_id |= MAGIC_VAL;
183 return msg_id;
184}
185
186static void mailbox_release_msgid(struct mailbox_channel *mb_chann, int msg_id)
187{
188 msg_id &= ~MAGIC_VAL_MASK;
189 xa_erase_irq(xa: &mb_chann->chan_xa, index: msg_id);
190}
191
192static void mailbox_release_msg(struct mailbox_channel *mb_chann,
193 struct mailbox_msg *mb_msg)
194{
195 MB_DBG(mb_chann, "msg_id 0x%x msg opcode 0x%x",
196 mb_msg->pkg.header.id, mb_msg->pkg.header.opcode);
197 if (mb_msg->notify_cb)
198 mb_msg->notify_cb(mb_msg->handle, NULL, 0);
199 kfree(objp: mb_msg);
200}
201
202static int
203mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg)
204{
205 void __iomem *write_addr;
206 u32 ringbuf_size;
207 u32 head, tail;
208 u32 start_addr;
209 u32 tmp_tail;
210
211 head = mailbox_get_headptr(mb_chann, type: CHAN_RES_X2I);
212 tail = mb_chann->x2i_tail;
213 ringbuf_size = mailbox_get_ringbuf_size(mb_chann, type: CHAN_RES_X2I);
214 start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr;
215 tmp_tail = tail + mb_msg->pkg_size;
216
217 if (tail < head && tmp_tail >= head)
218 goto no_space;
219
220 if (tail >= head && (tmp_tail > ringbuf_size - sizeof(u32) &&
221 mb_msg->pkg_size >= head))
222 goto no_space;
223
224 if (tail >= head && tmp_tail > ringbuf_size - sizeof(u32)) {
225 write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail;
226 writel(TOMBSTONE, addr: write_addr);
227
228 /* tombstone is set. Write from the start of the ringbuf */
229 tail = 0;
230 }
231
232 write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail;
233 memcpy_toio(write_addr, &mb_msg->pkg, mb_msg->pkg_size);
234 mailbox_set_tailptr(mb_chann, tailptr_val: tail + mb_msg->pkg_size);
235
236 trace_mbox_set_tail(MAILBOX_NAME, chann_id: mb_chann->msix_irq,
237 opcode: mb_msg->pkg.header.opcode,
238 id: mb_msg->pkg.header.id);
239
240 return 0;
241
242no_space:
243 return -ENOSPC;
244}
245
246static int
247mailbox_get_resp(struct mailbox_channel *mb_chann, struct xdna_msg_header *header,
248 void __iomem *data)
249{
250 struct mailbox_msg *mb_msg;
251 int msg_id;
252 int ret = 0;
253
254 msg_id = header->id;
255 if (!mailbox_validate_msgid(msg_id)) {
256 MB_ERR(mb_chann, "Bad message ID 0x%x", msg_id);
257 return -EINVAL;
258 }
259
260 msg_id &= ~MAGIC_VAL_MASK;
261 mb_msg = xa_erase_irq(xa: &mb_chann->chan_xa, index: msg_id);
262 if (!mb_msg) {
263 MB_ERR(mb_chann, "Cannot find msg 0x%x", msg_id);
264 return -EINVAL;
265 }
266
267 MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x",
268 header->opcode, header->total_size, header->id);
269 if (mb_msg->notify_cb) {
270 ret = mb_msg->notify_cb(mb_msg->handle, data, header->total_size);
271 if (unlikely(ret))
272 MB_ERR(mb_chann, "Message callback ret %d", ret);
273 }
274
275 kfree(objp: mb_msg);
276 return ret;
277}
278
279static int mailbox_get_msg(struct mailbox_channel *mb_chann)
280{
281 struct xdna_msg_header header;
282 void __iomem *read_addr;
283 u32 msg_size, rest;
284 u32 ringbuf_size;
285 u32 head, tail;
286 u32 start_addr;
287 int ret;
288
289 if (mailbox_reg_read_non_zero(mb_chann, mbox_reg: mb_chann->res[CHAN_RES_I2X].mb_tail_ptr_reg, val: &tail))
290 return -EINVAL;
291 head = mb_chann->i2x_head;
292 ringbuf_size = mailbox_get_ringbuf_size(mb_chann, type: CHAN_RES_I2X);
293 start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr;
294
295 if (unlikely(tail > ringbuf_size || !IS_ALIGNED(tail, 4))) {
296 MB_WARN_ONCE(mb_chann, "Invalid tail 0x%x", tail);
297 return -EINVAL;
298 }
299
300 /* ringbuf empty */
301 if (head == tail)
302 return -ENOENT;
303
304 if (head == ringbuf_size)
305 head = 0;
306
307 /* Peek size of the message or TOMBSTONE */
308 read_addr = mb_chann->mb->res.ringbuf_base + start_addr + head;
309 header.total_size = readl(addr: read_addr);
310 /* size is TOMBSTONE, set next read from 0 */
311 if (header.total_size == TOMBSTONE) {
312 if (head < tail) {
313 MB_WARN_ONCE(mb_chann, "Tombstone, head 0x%x tail 0x%x",
314 head, tail);
315 return -EINVAL;
316 }
317 mailbox_set_headptr(mb_chann, headptr_val: 0);
318 return 0;
319 }
320
321 if (unlikely(!header.total_size || !IS_ALIGNED(header.total_size, 4))) {
322 MB_WARN_ONCE(mb_chann, "Invalid total size 0x%x", header.total_size);
323 return -EINVAL;
324 }
325 msg_size = sizeof(header) + header.total_size;
326
327 if (msg_size > ringbuf_size - head || msg_size > tail - head) {
328 MB_WARN_ONCE(mb_chann, "Invalid message size %d, tail %d, head %d",
329 msg_size, tail, head);
330 return -EINVAL;
331 }
332
333 rest = sizeof(header) - sizeof(u32);
334 read_addr += sizeof(u32);
335 memcpy_fromio((u32 *)&header + 1, read_addr, rest);
336 read_addr += rest;
337
338 ret = mailbox_get_resp(mb_chann, header: &header, data: read_addr);
339
340 mailbox_set_headptr(mb_chann, headptr_val: head + msg_size);
341 /* After update head, it can equal to ringbuf_size. This is expected. */
342 trace_mbox_set_head(MAILBOX_NAME, chann_id: mb_chann->msix_irq,
343 opcode: header.opcode, id: header.id);
344
345 return ret;
346}
347
348static irqreturn_t mailbox_irq_handler(int irq, void *p)
349{
350 struct mailbox_channel *mb_chann = p;
351
352 trace_mbox_irq_handle(MAILBOX_NAME, irq);
353 /* Schedule a rx_work to call the callback functions */
354 queue_work(wq: mb_chann->work_q, work: &mb_chann->rx_work);
355
356 return IRQ_HANDLED;
357}
358
359static void mailbox_rx_worker(struct work_struct *rx_work)
360{
361 struct mailbox_channel *mb_chann;
362 int ret;
363
364 mb_chann = container_of(rx_work, struct mailbox_channel, rx_work);
365
366 if (READ_ONCE(mb_chann->bad_state)) {
367 MB_ERR(mb_chann, "Channel in bad state, work aborted");
368 return;
369 }
370
371again:
372 mailbox_reg_write(mb_chann, mbox_reg: mb_chann->iohub_int_addr, data: 0);
373
374 while (1) {
375 /*
376 * If return is 0, keep consuming next message, until there is
377 * no messages or an error happened.
378 */
379 ret = mailbox_get_msg(mb_chann);
380 if (ret == -ENOENT)
381 break;
382
383 /* Other error means device doesn't look good, disable irq. */
384 if (unlikely(ret)) {
385 MB_ERR(mb_chann, "Unexpected ret %d, disable irq", ret);
386 WRITE_ONCE(mb_chann->bad_state, true);
387 return;
388 }
389 }
390
391 /*
392 * The hardware will not generate interrupt if firmware creates a new
393 * response right after driver clears interrupt register. Check
394 * the interrupt register to make sure there is not any new response
395 * before exiting.
396 */
397 if (mailbox_reg_read(mb_chann, mbox_reg: mb_chann->iohub_int_addr))
398 goto again;
399}
400
401int xdna_mailbox_send_msg(struct mailbox_channel *mb_chann,
402 const struct xdna_mailbox_msg *msg, u64 tx_timeout)
403{
404 struct xdna_msg_header *header;
405 struct mailbox_msg *mb_msg;
406 size_t pkg_size;
407 int ret;
408
409 pkg_size = sizeof(*header) + msg->send_size;
410 if (pkg_size > mailbox_get_ringbuf_size(mb_chann, type: CHAN_RES_X2I)) {
411 MB_ERR(mb_chann, "Message size larger than ringbuf size");
412 return -EINVAL;
413 }
414
415 if (unlikely(!IS_ALIGNED(msg->send_size, 4))) {
416 MB_ERR(mb_chann, "Message must be 4 bytes align");
417 return -EINVAL;
418 }
419
420 /* The fist word in payload can NOT be TOMBSTONE */
421 if (unlikely(((u32 *)msg->send_data)[0] == TOMBSTONE)) {
422 MB_ERR(mb_chann, "Tomb stone in data");
423 return -EINVAL;
424 }
425
426 if (READ_ONCE(mb_chann->bad_state)) {
427 MB_ERR(mb_chann, "Channel in bad state");
428 return -EPIPE;
429 }
430
431 mb_msg = kzalloc(sizeof(*mb_msg) + pkg_size, GFP_KERNEL);
432 if (!mb_msg)
433 return -ENOMEM;
434
435 mb_msg->handle = msg->handle;
436 mb_msg->notify_cb = msg->notify_cb;
437 mb_msg->pkg_size = pkg_size;
438
439 header = &mb_msg->pkg.header;
440 /*
441 * Hardware use total_size and size to split huge message.
442 * We do not support it here. Thus the values are the same.
443 */
444 header->total_size = msg->send_size;
445 header->sz_ver = FIELD_PREP(MSG_BODY_SZ, msg->send_size) |
446 FIELD_PREP(MSG_PROTO_VER, MSG_PROTOCOL_VERSION);
447 header->opcode = msg->opcode;
448 memcpy(mb_msg->pkg.payload, msg->send_data, msg->send_size);
449
450 ret = mailbox_acquire_msgid(mb_chann, mb_msg);
451 if (unlikely(ret < 0)) {
452 MB_ERR(mb_chann, "mailbox_acquire_msgid failed");
453 goto msg_id_failed;
454 }
455 header->id = ret;
456
457 MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x",
458 header->opcode, header->total_size, header->id);
459
460 ret = mailbox_send_msg(mb_chann, mb_msg);
461 if (ret) {
462 MB_DBG(mb_chann, "Error in mailbox send msg, ret %d", ret);
463 goto release_id;
464 }
465
466 return 0;
467
468release_id:
469 mailbox_release_msgid(mb_chann, msg_id: header->id);
470msg_id_failed:
471 kfree(objp: mb_msg);
472 return ret;
473}
474
475struct mailbox_channel *
476xdna_mailbox_create_channel(struct mailbox *mb,
477 const struct xdna_mailbox_chann_res *x2i,
478 const struct xdna_mailbox_chann_res *i2x,
479 u32 iohub_int_addr,
480 int mb_irq)
481{
482 struct mailbox_channel *mb_chann;
483 int ret;
484
485 if (!is_power_of_2(n: x2i->rb_size) || !is_power_of_2(n: i2x->rb_size)) {
486 pr_err("Ring buf size must be power of 2");
487 return NULL;
488 }
489
490 mb_chann = kzalloc(sizeof(*mb_chann), GFP_KERNEL);
491 if (!mb_chann)
492 return NULL;
493
494 mb_chann->mb = mb;
495 mb_chann->msix_irq = mb_irq;
496 mb_chann->iohub_int_addr = iohub_int_addr;
497 memcpy(&mb_chann->res[CHAN_RES_X2I], x2i, sizeof(*x2i));
498 memcpy(&mb_chann->res[CHAN_RES_I2X], i2x, sizeof(*i2x));
499
500 xa_init_flags(xa: &mb_chann->chan_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
501 mb_chann->x2i_tail = mailbox_get_tailptr(mb_chann, type: CHAN_RES_X2I);
502 mb_chann->i2x_head = mailbox_get_headptr(mb_chann, type: CHAN_RES_I2X);
503
504 INIT_WORK(&mb_chann->rx_work, mailbox_rx_worker);
505 mb_chann->work_q = create_singlethread_workqueue(MAILBOX_NAME);
506 if (!mb_chann->work_q) {
507 MB_ERR(mb_chann, "Create workqueue failed");
508 goto free_and_out;
509 }
510
511 /* Everything look good. Time to enable irq handler */
512 ret = request_irq(irq: mb_irq, handler: mailbox_irq_handler, flags: 0, MAILBOX_NAME, dev: mb_chann);
513 if (ret) {
514 MB_ERR(mb_chann, "Failed to request irq %d ret %d", mb_irq, ret);
515 goto destroy_wq;
516 }
517
518 mb_chann->bad_state = false;
519 mailbox_reg_write(mb_chann, mbox_reg: mb_chann->iohub_int_addr, data: 0);
520
521 MB_DBG(mb_chann, "Mailbox channel created (irq: %d)", mb_chann->msix_irq);
522 return mb_chann;
523
524destroy_wq:
525 destroy_workqueue(wq: mb_chann->work_q);
526free_and_out:
527 kfree(objp: mb_chann);
528 return NULL;
529}
530
531int xdna_mailbox_destroy_channel(struct mailbox_channel *mb_chann)
532{
533 struct mailbox_msg *mb_msg;
534 unsigned long msg_id;
535
536 MB_DBG(mb_chann, "IRQ disabled and RX work cancelled");
537 free_irq(mb_chann->msix_irq, mb_chann);
538 destroy_workqueue(wq: mb_chann->work_q);
539 /* We can clean up and release resources */
540
541 xa_for_each(&mb_chann->chan_xa, msg_id, mb_msg)
542 mailbox_release_msg(mb_chann, mb_msg);
543
544 xa_destroy(&mb_chann->chan_xa);
545
546 MB_DBG(mb_chann, "Mailbox channel destroyed, irq: %d", mb_chann->msix_irq);
547 kfree(objp: mb_chann);
548 return 0;
549}
550
551void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann)
552{
553 /* Disable an irq and wait. This might sleep. */
554 disable_irq(irq: mb_chann->msix_irq);
555
556 /* Cancel RX work and wait for it to finish */
557 cancel_work_sync(work: &mb_chann->rx_work);
558 MB_DBG(mb_chann, "IRQ disabled and RX work cancelled");
559}
560
561struct mailbox *xdnam_mailbox_create(struct drm_device *ddev,
562 const struct xdna_mailbox_res *res)
563{
564 struct mailbox *mb;
565
566 mb = drmm_kzalloc(dev: ddev, size: sizeof(*mb), GFP_KERNEL);
567 if (!mb)
568 return NULL;
569 mb->dev = ddev->dev;
570
571 /* mailbox and ring buf base and size information */
572 memcpy(&mb->res, res, sizeof(*res));
573
574 return mb;
575}
576

source code of linux/drivers/accel/amdxdna/amdxdna_mailbox.c