1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright 2018-2020 Broadcom.
4 */
5
6#include <linux/delay.h>
7#include <linux/fs.h>
8#include <linux/hash.h>
9#include <linux/interrupt.h>
10#include <linux/list.h>
11#include <linux/module.h>
12#include <linux/poll.h>
13#include <linux/sizes.h>
14#include <linux/spinlock.h>
15#include <linux/timer.h>
16
17#include "bcm_vk.h"
18#include "bcm_vk_msg.h"
19#include "bcm_vk_sg.h"
20
21/* functions to manipulate the transport id in msg block */
22#define BCM_VK_MSG_Q_SHIFT 4
23#define BCM_VK_MSG_Q_MASK 0xF
24#define BCM_VK_MSG_ID_MASK 0xFFF
25
26#define BCM_VK_DMA_DRAIN_MAX_MS 2000
27
28/* number x q_size will be the max number of msg processed per loop */
29#define BCM_VK_MSG_PROC_MAX_LOOP 2
30
31/* module parameter */
32static bool hb_mon = true;
33module_param(hb_mon, bool, 0444);
34MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n");
35static int batch_log = 1;
36module_param(batch_log, int, 0444);
37MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n");
38
39static bool hb_mon_is_on(void)
40{
41 return hb_mon;
42}
43
44static u32 get_q_num(const struct vk_msg_blk *msg)
45{
46 u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK;
47
48 if (q_num >= VK_MSGQ_PER_CHAN_MAX)
49 q_num = VK_MSGQ_NUM_DEFAULT;
50 return q_num;
51}
52
53static void set_q_num(struct vk_msg_blk *msg, u32 q_num)
54{
55 u32 trans_q;
56
57 if (q_num >= VK_MSGQ_PER_CHAN_MAX)
58 trans_q = VK_MSGQ_NUM_DEFAULT;
59 else
60 trans_q = q_num;
61
62 msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q;
63}
64
65static u32 get_msg_id(const struct vk_msg_blk *msg)
66{
67 return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK);
68}
69
70static void set_msg_id(struct vk_msg_blk *msg, u32 val)
71{
72 msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg);
73}
74
75static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc)
76{
77 return ((idx + inc) & qinfo->q_mask);
78}
79
80static
81struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo,
82 u32 idx)
83{
84 return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx);
85}
86
87static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq,
88 const struct bcm_vk_sync_qinfo *qinfo)
89{
90 u32 wr_idx, rd_idx;
91
92 wr_idx = readl_relaxed(&msgq->wr_idx);
93 rd_idx = readl_relaxed(&msgq->rd_idx);
94
95 return ((wr_idx - rd_idx) & qinfo->q_mask);
96}
97
98static
99u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq,
100 const struct bcm_vk_sync_qinfo *qinfo)
101{
102 return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1);
103}
104
105/* number of retries when enqueue message fails before returning EAGAIN */
106#define BCM_VK_H2VK_ENQ_RETRY 10
107#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50
108
109bool bcm_vk_drv_access_ok(struct bcm_vk *vk)
110{
111 return (!!atomic_read(v: &vk->msgq_inited));
112}
113
114void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)
115{
116 struct bcm_vk_alert *alert = &vk->host_alert;
117 unsigned long flags;
118
119 /* use irqsave version as this maybe called inside timer interrupt */
120 spin_lock_irqsave(&vk->host_alert_lock, flags);
121 alert->notfs |= bit_mask;
122 spin_unlock_irqrestore(lock: &vk->host_alert_lock, flags);
123
124 if (test_and_set_bit(nr: BCM_VK_WQ_NOTF_PEND, addr: vk->wq_offload) == 0)
125 queue_work(wq: vk->wq_thread, work: &vk->wq_work);
126}
127
128/*
129 * Heartbeat related defines
130 * The heartbeat from host is a last resort. If stuck condition happens
131 * on the card, firmware is supposed to detect it. Therefore, the heartbeat
132 * values used will be more relaxed on the driver, which need to be bigger
133 * than the watchdog timeout on the card. The watchdog timeout on the card
134 * is 20s, with a jitter of 2s => 22s. We use a value of 27s here.
135 */
136#define BCM_VK_HB_TIMER_S 3
137#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)
138#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S)
139
140static void bcm_vk_hb_poll(struct work_struct *work)
141{
142 u32 uptime_s;
143 struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl,
144 work);
145 struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);
146
147 if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) {
148 /* read uptime from register and compare */
149 uptime_s = vkread32(vk, bar: BAR_0, BAR_OS_UPTIME);
150
151 if (uptime_s == hb->last_uptime)
152 hb->lost_cnt++;
153 else /* reset to avoid accumulation */
154 hb->lost_cnt = 0;
155
156 dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n",
157 hb->last_uptime, uptime_s, hb->lost_cnt);
158
159 /*
160 * if the interface goes down without any activity, a value
161 * of 0xFFFFFFFF will be continuously read, and the detection
162 * will be happened eventually.
163 */
164 hb->last_uptime = uptime_s;
165 } else {
166 /* reset heart beat lost cnt */
167 hb->lost_cnt = 0;
168 }
169
170 /* next, check if heartbeat exceeds limit */
171 if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) {
172 dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n",
173 BCM_VK_HB_LOST_MAX,
174 BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S);
175
176 bcm_vk_blk_drv_access(vk);
177 bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL);
178 }
179 /* re-arm timer */
180 schedule_delayed_work(dwork: &hb->work, BCM_VK_HB_TIMER_VALUE);
181}
182
183void bcm_vk_hb_init(struct bcm_vk *vk)
184{
185 struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;
186
187 INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll);
188 schedule_delayed_work(dwork: &hb->work, BCM_VK_HB_TIMER_VALUE);
189}
190
191void bcm_vk_hb_deinit(struct bcm_vk *vk)
192{
193 struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;
194
195 cancel_delayed_work_sync(dwork: &hb->work);
196}
197
198static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,
199 unsigned int start,
200 unsigned int nbits)
201{
202 spin_lock(lock: &vk->msg_id_lock);
203 bitmap_clear(map: vk->bmap, start, nbits);
204 spin_unlock(lock: &vk->msg_id_lock);
205}
206
207/*
208 * allocate a ctx per file struct
209 */
210static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid)
211{
212 u32 i;
213 struct bcm_vk_ctx *ctx = NULL;
214 u32 hash_idx = hash_32(val: pid, VK_PID_HT_SHIFT_BIT);
215
216 spin_lock(lock: &vk->ctx_lock);
217
218 /* check if it is in reset, if so, don't allow */
219 if (vk->reset_pid) {
220 dev_err(&vk->pdev->dev,
221 "No context allowed during reset by pid %d\n",
222 vk->reset_pid);
223
224 goto in_reset_exit;
225 }
226
227 for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {
228 if (!vk->ctx[i].in_use) {
229 vk->ctx[i].in_use = true;
230 ctx = &vk->ctx[i];
231 break;
232 }
233 }
234
235 if (!ctx) {
236 dev_err(&vk->pdev->dev, "All context in use\n");
237
238 goto all_in_use_exit;
239 }
240
241 /* set the pid and insert it to hash table */
242 ctx->pid = pid;
243 ctx->hash_idx = hash_idx;
244 list_add_tail(new: &ctx->node, head: &vk->pid_ht[hash_idx].head);
245
246 /* increase kref */
247 kref_get(kref: &vk->kref);
248
249 /* clear counter */
250 atomic_set(v: &ctx->pend_cnt, i: 0);
251 atomic_set(v: &ctx->dma_cnt, i: 0);
252 init_waitqueue_head(&ctx->rd_wq);
253
254all_in_use_exit:
255in_reset_exit:
256 spin_unlock(lock: &vk->ctx_lock);
257
258 return ctx;
259}
260
261static u16 bcm_vk_get_msg_id(struct bcm_vk *vk)
262{
263 u16 rc = VK_MSG_ID_OVERFLOW;
264 u16 test_bit_count = 0;
265
266 spin_lock(lock: &vk->msg_id_lock);
267 while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) {
268 /*
269 * first time come in this loop, msg_id will be 0
270 * and the first one tested will be 1. We skip
271 * VK_SIMPLEX_MSG_ID (0) for one way host2vk
272 * communication
273 */
274 vk->msg_id++;
275 if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE)
276 vk->msg_id = 1;
277
278 if (test_bit(vk->msg_id, vk->bmap)) {
279 test_bit_count++;
280 continue;
281 }
282 rc = vk->msg_id;
283 bitmap_set(map: vk->bmap, start: vk->msg_id, nbits: 1);
284 break;
285 }
286 spin_unlock(lock: &vk->msg_id_lock);
287
288 return rc;
289}
290
291static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx)
292{
293 u32 idx;
294 u32 hash_idx;
295 pid_t pid;
296 struct bcm_vk_ctx *entry;
297 int count = 0;
298
299 if (!ctx) {
300 dev_err(&vk->pdev->dev, "NULL context detected\n");
301 return -EINVAL;
302 }
303 idx = ctx->idx;
304 pid = ctx->pid;
305
306 spin_lock(lock: &vk->ctx_lock);
307
308 if (!vk->ctx[idx].in_use) {
309 dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx);
310 } else {
311 vk->ctx[idx].in_use = false;
312 vk->ctx[idx].miscdev = NULL;
313
314 /* Remove it from hash list and see if it is the last one. */
315 list_del(entry: &ctx->node);
316 hash_idx = ctx->hash_idx;
317 list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) {
318 if (entry->pid == pid)
319 count++;
320 }
321 }
322
323 spin_unlock(lock: &vk->ctx_lock);
324
325 return count;
326}
327
328static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry)
329{
330 int proc_cnt;
331
332 bcm_vk_sg_free(dev, dma: entry->dma, VK_DMA_MAX_ADDRS, proc_cnt: &proc_cnt);
333 if (proc_cnt)
334 atomic_dec(v: &entry->ctx->dma_cnt);
335
336 kfree(objp: entry->to_h_msg);
337 kfree(objp: entry);
338}
339
340static void bcm_vk_drain_all_pend(struct device *dev,
341 struct bcm_vk_msg_chan *chan,
342 struct bcm_vk_ctx *ctx)
343{
344 u32 num;
345 struct bcm_vk_wkent *entry, *tmp;
346 struct bcm_vk *vk;
347 struct list_head del_q;
348
349 if (ctx)
350 vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
351
352 INIT_LIST_HEAD(list: &del_q);
353 spin_lock(lock: &chan->pendq_lock);
354 for (num = 0; num < chan->q_nr; num++) {
355 list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) {
356 if ((!ctx) || (entry->ctx->idx == ctx->idx)) {
357 list_move_tail(list: &entry->node, head: &del_q);
358 }
359 }
360 }
361 spin_unlock(lock: &chan->pendq_lock);
362
363 /* batch clean up */
364 num = 0;
365 list_for_each_entry_safe(entry, tmp, &del_q, node) {
366 list_del(entry: &entry->node);
367 num++;
368 if (ctx) {
369 struct vk_msg_blk *msg;
370 int bit_set;
371 bool responded;
372 u32 msg_id;
373
374 /* if it is specific ctx, log for any stuck */
375 msg = entry->to_v_msg;
376 msg_id = get_msg_id(msg);
377 bit_set = test_bit(msg_id, vk->bmap);
378 responded = entry->to_h_msg ? true : false;
379 if (num <= batch_log)
380 dev_info(dev,
381 "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n",
382 msg->function_id, msg->size,
383 msg_id, entry->seq_num,
384 msg->context_id, entry->ctx->idx,
385 msg->cmd, msg->arg,
386 responded ? "T" : "F", bit_set);
387 if (responded)
388 atomic_dec(v: &ctx->pend_cnt);
389 else if (bit_set)
390 bcm_vk_msgid_bitmap_clear(vk, start: msg_id, nbits: 1);
391 }
392 bcm_vk_free_wkent(dev, entry);
393 }
394 if (num && ctx)
395 dev_info(dev, "Total drained items %d [fd-%d]\n",
396 num, ctx->idx);
397}
398
399void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk)
400{
401 bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_v_msg_chan, NULL);
402 bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_h_msg_chan, NULL);
403}
404
405/*
406 * Function to sync up the messages queue info that is provided by BAR1
407 */
408int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync)
409{
410 struct bcm_vk_msgq __iomem *msgq;
411 struct device *dev = &vk->pdev->dev;
412 u32 msgq_off;
413 u32 num_q;
414 struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan,
415 &vk->to_h_msg_chan};
416 struct bcm_vk_msg_chan *chan;
417 int i, j;
418 int ret = 0;
419
420 /*
421 * If the driver is loaded at startup where vk OS is not up yet,
422 * the msgq-info may not be available until a later time. In
423 * this case, we skip and the sync function is supposed to be
424 * called again.
425 */
426 if (!bcm_vk_msgq_marker_valid(vk)) {
427 dev_info(dev, "BAR1 msgq marker not initialized.\n");
428 return -EAGAIN;
429 }
430
431 msgq_off = vkread32(vk, bar: BAR_1, VK_BAR1_MSGQ_CTRL_OFF);
432
433 /* each side is always half the total */
434 num_q = vkread32(vk, bar: BAR_1, VK_BAR1_MSGQ_NR) / 2;
435 if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) {
436 dev_err(dev,
437 "Advertised msgq %d error - max %d allowed\n",
438 num_q, VK_MSGQ_PER_CHAN_MAX);
439 return -EINVAL;
440 }
441
442 vk->to_v_msg_chan.q_nr = num_q;
443 vk->to_h_msg_chan.q_nr = num_q;
444
445 /* first msgq location */
446 msgq = vk->bar[BAR_1] + msgq_off;
447
448 /*
449 * if this function is called when it is already inited,
450 * something is wrong
451 */
452 if (bcm_vk_drv_access_ok(vk) && !force_sync) {
453 dev_err(dev, "Msgq info already in sync\n");
454 return -EPERM;
455 }
456
457 for (i = 0; i < ARRAY_SIZE(chan_list); i++) {
458 chan = chan_list[i];
459 memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo));
460
461 for (j = 0; j < num_q; j++) {
462 struct bcm_vk_sync_qinfo *qinfo;
463 u32 msgq_start;
464 u32 msgq_size;
465 u32 msgq_nxt;
466 u32 msgq_db_offset, q_db_offset;
467
468 chan->msgq[j] = msgq;
469 msgq_start = readl_relaxed(&msgq->start);
470 msgq_size = readl_relaxed(&msgq->size);
471 msgq_nxt = readl_relaxed(&msgq->nxt);
472 msgq_db_offset = readl_relaxed(&msgq->db_offset);
473 q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1));
474 if (q_db_offset == (~msgq_db_offset >> DB_SHIFT))
475 msgq_db_offset = q_db_offset;
476 else
477 /* fall back to default */
478 msgq_db_offset = VK_BAR0_Q_DB_BASE(j);
479
480 dev_info(dev,
481 "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n",
482 j,
483 readw_relaxed(&msgq->type),
484 readw_relaxed(&msgq->num),
485 msgq_start,
486 msgq_db_offset,
487 readl_relaxed(&msgq->rd_idx),
488 readl_relaxed(&msgq->wr_idx),
489 msgq_size,
490 msgq_nxt);
491
492 qinfo = &chan->sync_qinfo[j];
493 /* formulate and record static info */
494 qinfo->q_start = vk->bar[BAR_1] + msgq_start;
495 qinfo->q_size = msgq_size;
496 /* set low threshold as 50% or 1/2 */
497 qinfo->q_low = qinfo->q_size >> 1;
498 qinfo->q_mask = qinfo->q_size - 1;
499 qinfo->q_db_offset = msgq_db_offset;
500
501 msgq++;
502 }
503 }
504 atomic_set(v: &vk->msgq_inited, i: 1);
505
506 return ret;
507}
508
509static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan)
510{
511 u32 i;
512
513 mutex_init(&chan->msgq_mutex);
514 spin_lock_init(&chan->pendq_lock);
515 for (i = 0; i < VK_MSGQ_MAX_NR; i++)
516 INIT_LIST_HEAD(list: &chan->pendq[i]);
517
518 return 0;
519}
520
521static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num,
522 struct bcm_vk_wkent *entry)
523{
524 struct bcm_vk_ctx *ctx;
525
526 spin_lock(lock: &chan->pendq_lock);
527 list_add_tail(new: &entry->node, head: &chan->pendq[q_num]);
528 if (entry->to_h_msg) {
529 ctx = entry->ctx;
530 atomic_inc(v: &ctx->pend_cnt);
531 wake_up_interruptible(&ctx->rd_wq);
532 }
533 spin_unlock(lock: &chan->pendq_lock);
534}
535
536static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk,
537 struct bcm_vk_wkent *entry,
538 struct _vk_data *data,
539 unsigned int num_planes)
540{
541 unsigned int i;
542 unsigned int item_cnt = 0;
543 struct device *dev = &vk->pdev->dev;
544 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
545 struct vk_msg_blk *msg = &entry->to_v_msg[0];
546 struct bcm_vk_msgq __iomem *msgq;
547 struct bcm_vk_sync_qinfo *qinfo;
548 u32 ib_sgl_size = 0;
549 u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks];
550 u32 avail;
551 u32 q_num;
552
553 /* check if high watermark is hit, and if so, skip */
554 q_num = get_q_num(msg);
555 msgq = chan->msgq[q_num];
556 qinfo = &chan->sync_qinfo[q_num];
557 avail = msgq_avail_space(msgq, qinfo);
558 if (avail < qinfo->q_low) {
559 dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n",
560 avail, qinfo->q_size);
561 return 0;
562 }
563
564 for (i = 0; i < num_planes; i++) {
565 if (data[i].address &&
566 (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) {
567 item_cnt++;
568 memcpy(buf, entry->dma[i].sglist, data[i].size);
569 ib_sgl_size += data[i].size;
570 buf += data[i].size;
571 }
572 }
573
574 dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n",
575 item_cnt, ib_sgl_size, vk->ib_sgl_size);
576
577 /* round up size */
578 ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1)
579 >> VK_MSGQ_BLK_SZ_SHIFT;
580
581 return ib_sgl_size;
582}
583
584void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val)
585{
586 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
587 struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num];
588
589 vkwrite32(vk, value: db_val, bar: BAR_0, offset: qinfo->q_db_offset);
590}
591
592static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry)
593{
594 static u32 seq_num;
595 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;
596 struct device *dev = &vk->pdev->dev;
597 struct vk_msg_blk *src = &entry->to_v_msg[0];
598
599 struct vk_msg_blk __iomem *dst;
600 struct bcm_vk_msgq __iomem *msgq;
601 struct bcm_vk_sync_qinfo *qinfo;
602 u32 q_num = get_q_num(msg: src);
603 u32 wr_idx; /* local copy */
604 u32 i;
605 u32 avail;
606 u32 retry;
607
608 if (entry->to_v_blks != src->size + 1) {
609 dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n",
610 entry->to_v_blks,
611 src->size + 1,
612 get_msg_id(src),
613 src->function_id,
614 src->context_id);
615 return -EMSGSIZE;
616 }
617
618 msgq = chan->msgq[q_num];
619 qinfo = &chan->sync_qinfo[q_num];
620
621 mutex_lock(&chan->msgq_mutex);
622
623 avail = msgq_avail_space(msgq, qinfo);
624
625 /* if not enough space, return EAGAIN and let app handles it */
626 retry = 0;
627 while ((avail < entry->to_v_blks) &&
628 (retry++ < BCM_VK_H2VK_ENQ_RETRY)) {
629 mutex_unlock(lock: &chan->msgq_mutex);
630
631 msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS);
632 mutex_lock(&chan->msgq_mutex);
633 avail = msgq_avail_space(msgq, qinfo);
634 }
635 if (retry > BCM_VK_H2VK_ENQ_RETRY) {
636 mutex_unlock(lock: &chan->msgq_mutex);
637 return -EAGAIN;
638 }
639
640 /* at this point, mutex is taken and there is enough space */
641 entry->seq_num = seq_num++; /* update debug seq number */
642 wr_idx = readl_relaxed(&msgq->wr_idx);
643
644 if (wr_idx >= qinfo->q_size) {
645 dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!",
646 wr_idx, qinfo->q_size);
647 bcm_vk_blk_drv_access(vk);
648 bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN);
649 goto idx_err;
650 }
651
652 dst = msgq_blk_addr(qinfo, idx: wr_idx);
653 for (i = 0; i < entry->to_v_blks; i++) {
654 memcpy_toio(dst, src, sizeof(*dst));
655
656 src++;
657 wr_idx = msgq_inc(qinfo, idx: wr_idx, inc: 1);
658 dst = msgq_blk_addr(qinfo, idx: wr_idx);
659 }
660
661 /* flush the write pointer */
662 writel(val: wr_idx, addr: &msgq->wr_idx);
663
664 /* log new info for debugging */
665 dev_dbg(dev,
666 "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n",
667 readl_relaxed(&msgq->num),
668 readl_relaxed(&msgq->rd_idx),
669 wr_idx,
670 entry->to_v_blks,
671 msgq_occupied(msgq, qinfo),
672 msgq_avail_space(msgq, qinfo),
673 readl_relaxed(&msgq->size));
674 /*
675 * press door bell based on queue number. 1 is added to the wr_idx
676 * to avoid the value of 0 appearing on the VK side to distinguish
677 * from initial value.
678 */
679 bcm_to_v_q_doorbell(vk, q_num, db_val: wr_idx + 1);
680idx_err:
681 mutex_unlock(lock: &chan->msgq_mutex);
682 return 0;
683}
684
685int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type,
686 const pid_t pid, const u32 q_num)
687{
688 int rc = 0;
689 struct bcm_vk_wkent *entry;
690 struct device *dev = &vk->pdev->dev;
691
692 /*
693 * check if the marker is still good. Sometimes, the PCIe interface may
694 * have gone done, and if so and we ship down thing based on broken
695 * values, kernel may panic.
696 */
697 if (!bcm_vk_msgq_marker_valid(vk)) {
698 dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n",
699 vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY));
700 return -EINVAL;
701 }
702
703 entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL);
704 if (!entry)
705 return -ENOMEM;
706 entry->to_v_blks = 1; /* always 1 block */
707
708 /* fill up necessary data */
709 entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN;
710 set_q_num(msg: &entry->to_v_msg[0], q_num);
711 set_msg_id(msg: &entry->to_v_msg[0], VK_SIMPLEX_MSG_ID);
712
713 entry->to_v_msg[0].cmd = shut_type;
714 entry->to_v_msg[0].arg = pid;
715
716 rc = bcm_to_v_msg_enqueue(vk, entry);
717 if (rc)
718 dev_err(dev,
719 "Sending shutdown message to q %d for pid %d fails.\n",
720 get_q_num(&entry->to_v_msg[0]), pid);
721
722 kfree(objp: entry);
723
724 return rc;
725}
726
727static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid,
728 const u32 q_num)
729{
730 int rc = 0;
731 struct device *dev = &vk->pdev->dev;
732
733 /*
734 * don't send down or do anything if message queue is not initialized
735 * and if it is the reset session, clear it.
736 */
737 if (!bcm_vk_drv_access_ok(vk)) {
738 if (vk->reset_pid == pid)
739 vk->reset_pid = 0;
740 return -EPERM;
741 }
742
743 dev_dbg(dev, "No more sessions, shut down pid %d\n", pid);
744
745 /* only need to do it if it is not the reset process */
746 if (vk->reset_pid != pid)
747 rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num);
748 else
749 /* put reset_pid to 0 if it is exiting last session */
750 vk->reset_pid = 0;
751
752 return rc;
753}
754
755static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk,
756 struct bcm_vk_msg_chan *chan,
757 u16 q_num,
758 u16 msg_id)
759{
760 struct bcm_vk_wkent *entry = NULL, *iter;
761
762 spin_lock(lock: &chan->pendq_lock);
763 list_for_each_entry(iter, &chan->pendq[q_num], node) {
764 if (get_msg_id(msg: &iter->to_v_msg[0]) == msg_id) {
765 list_del(entry: &iter->node);
766 entry = iter;
767 bcm_vk_msgid_bitmap_clear(vk, start: msg_id, nbits: 1);
768 break;
769 }
770 }
771 spin_unlock(lock: &chan->pendq_lock);
772 return entry;
773}
774
775s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk)
776{
777 struct device *dev = &vk->pdev->dev;
778 struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;
779 struct vk_msg_blk *data;
780 struct vk_msg_blk __iomem *src;
781 struct vk_msg_blk *dst;
782 struct bcm_vk_msgq __iomem *msgq;
783 struct bcm_vk_sync_qinfo *qinfo;
784 struct bcm_vk_wkent *entry;
785 u32 rd_idx, wr_idx;
786 u32 q_num, msg_id, j;
787 u32 num_blks;
788 s32 total = 0;
789 int cnt = 0;
790 int msg_processed = 0;
791 int max_msg_to_process;
792 bool exit_loop;
793
794 /*
795 * drain all the messages from the queues, and find its pending
796 * entry in the to_v queue, based on msg_id & q_num, and move the
797 * entry to the to_h pending queue, waiting for user space
798 * program to extract
799 */
800 mutex_lock(&chan->msgq_mutex);
801
802 for (q_num = 0; q_num < chan->q_nr; q_num++) {
803 msgq = chan->msgq[q_num];
804 qinfo = &chan->sync_qinfo[q_num];
805 max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size;
806
807 rd_idx = readl_relaxed(&msgq->rd_idx);
808 wr_idx = readl_relaxed(&msgq->wr_idx);
809 msg_processed = 0;
810 exit_loop = false;
811 while ((rd_idx != wr_idx) && !exit_loop) {
812 u8 src_size;
813
814 /*
815 * Make a local copy and get pointer to src blk
816 * The rd_idx is masked before getting the pointer to
817 * avoid out of bound access in case the interface goes
818 * down. It will end up pointing to the last block in
819 * the buffer, but subsequent src->size check would be
820 * able to catch this.
821 */
822 src = msgq_blk_addr(qinfo, idx: rd_idx & qinfo->q_mask);
823 src_size = readb(addr: &src->size);
824
825 if ((rd_idx >= qinfo->q_size) ||
826 (src_size > (qinfo->q_size - 1))) {
827 dev_crit(dev,
828 "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!",
829 rd_idx, src_size, qinfo->q_size);
830 bcm_vk_blk_drv_access(vk);
831 bcm_vk_set_host_alert(vk,
832 ERR_LOG_HOST_PCIE_DWN);
833 goto idx_err;
834 }
835
836 num_blks = src_size + 1;
837 data = kzalloc(size: num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL);
838 if (data) {
839 /* copy messages and linearize it */
840 dst = data;
841 for (j = 0; j < num_blks; j++) {
842 memcpy_fromio(dst, src, sizeof(*dst));
843
844 dst++;
845 rd_idx = msgq_inc(qinfo, idx: rd_idx, inc: 1);
846 src = msgq_blk_addr(qinfo, idx: rd_idx);
847 }
848 total++;
849 } else {
850 /*
851 * if we could not allocate memory in kernel,
852 * that is fatal.
853 */
854 dev_crit(dev, "Kernel mem allocation failure.\n");
855 total = -ENOMEM;
856 goto idx_err;
857 }
858
859 /* flush rd pointer after a message is dequeued */
860 writel(val: rd_idx, addr: &msgq->rd_idx);
861
862 /* log new info for debugging */
863 dev_dbg(dev,
864 "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n",
865 readl_relaxed(&msgq->num),
866 rd_idx,
867 wr_idx,
868 num_blks,
869 msgq_occupied(msgq, qinfo),
870 msgq_avail_space(msgq, qinfo),
871 readl_relaxed(&msgq->size));
872
873 /*
874 * No need to search if it is an autonomous one-way
875 * message from driver, as these messages do not bear
876 * a to_v pending item. Currently, only the shutdown
877 * message falls into this category.
878 */
879 if (data->function_id == VK_FID_SHUTDOWN) {
880 kfree(objp: data);
881 continue;
882 }
883
884 msg_id = get_msg_id(msg: data);
885 /* lookup original message in to_v direction */
886 entry = bcm_vk_dequeue_pending(vk,
887 chan: &vk->to_v_msg_chan,
888 q_num,
889 msg_id);
890
891 /*
892 * if there is message to does not have prior send,
893 * this is the location to add here
894 */
895 if (entry) {
896 entry->to_h_blks = num_blks;
897 entry->to_h_msg = data;
898 bcm_vk_append_pendq(chan: &vk->to_h_msg_chan,
899 q_num, entry);
900
901 } else {
902 if (cnt++ < batch_log)
903 dev_info(dev,
904 "Could not find MsgId[0x%x] for resp func %d bmap %d\n",
905 msg_id, data->function_id,
906 test_bit(msg_id, vk->bmap));
907 kfree(objp: data);
908 }
909 /* Fetch wr_idx to handle more back-to-back events */
910 wr_idx = readl(addr: &msgq->wr_idx);
911
912 /*
913 * cap the max so that even we try to handle more back-to-back events,
914 * so that it won't hold CPU too long or in case rd/wr idexes are
915 * corrupted which triggers infinite looping.
916 */
917 if (++msg_processed >= max_msg_to_process) {
918 dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n",
919 q_num, max_msg_to_process);
920 exit_loop = true;
921 }
922 }
923 }
924idx_err:
925 mutex_unlock(lock: &chan->msgq_mutex);
926 dev_dbg(dev, "total %d drained from queues\n", total);
927
928 return total;
929}
930
931/*
932 * init routine for all required data structures
933 */
934static int bcm_vk_data_init(struct bcm_vk *vk)
935{
936 int i;
937
938 spin_lock_init(&vk->ctx_lock);
939 for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {
940 vk->ctx[i].in_use = false;
941 vk->ctx[i].idx = i; /* self identity */
942 vk->ctx[i].miscdev = NULL;
943 }
944 spin_lock_init(&vk->msg_id_lock);
945 spin_lock_init(&vk->host_alert_lock);
946 vk->msg_id = 0;
947
948 /* initialize hash table */
949 for (i = 0; i < VK_PID_HT_SZ; i++)
950 INIT_LIST_HEAD(list: &vk->pid_ht[i].head);
951
952 return 0;
953}
954
955irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id)
956{
957 struct bcm_vk *vk = dev_id;
958
959 if (!bcm_vk_drv_access_ok(vk)) {
960 dev_err(&vk->pdev->dev,
961 "Interrupt %d received when msgq not inited\n", irq);
962 goto skip_schedule_work;
963 }
964
965 queue_work(wq: vk->wq_thread, work: &vk->wq_work);
966
967skip_schedule_work:
968 return IRQ_HANDLED;
969}
970
971int bcm_vk_open(struct inode *inode, struct file *p_file)
972{
973 struct bcm_vk_ctx *ctx;
974 struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data;
975 struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev);
976 struct device *dev = &vk->pdev->dev;
977 int rc = 0;
978
979 /* get a context and set it up for file */
980 ctx = bcm_vk_get_ctx(vk, pid: task_tgid_nr(current));
981 if (!ctx) {
982 dev_err(dev, "Error allocating context\n");
983 rc = -ENOMEM;
984 } else {
985 /*
986 * set up context and replace private data with context for
987 * other methods to use. Reason for the context is because
988 * it is allowed for multiple sessions to open the sysfs, and
989 * for each file open, when upper layer query the response,
990 * only those that are tied to a specific open should be
991 * returned. The context->idx will be used for such binding
992 */
993 ctx->miscdev = miscdev;
994 p_file->private_data = ctx;
995 dev_dbg(dev, "ctx_returned with idx %d, pid %d\n",
996 ctx->idx, ctx->pid);
997 }
998 return rc;
999}
1000
1001ssize_t bcm_vk_read(struct file *p_file,
1002 char __user *buf,
1003 size_t count,
1004 loff_t *f_pos)
1005{
1006 ssize_t rc = -ENOMSG;
1007 struct bcm_vk_ctx *ctx = p_file->private_data;
1008 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,
1009 miscdev);
1010 struct device *dev = &vk->pdev->dev;
1011 struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;
1012 struct bcm_vk_wkent *entry = NULL, *iter;
1013 u32 q_num;
1014 u32 rsp_length;
1015
1016 if (!bcm_vk_drv_access_ok(vk))
1017 return -EPERM;
1018
1019 dev_dbg(dev, "Buf count %zu\n", count);
1020
1021 /*
1022 * search through the pendq on the to_h chan, and return only those
1023 * that belongs to the same context. Search is always from the high to
1024 * the low priority queues
1025 */
1026 spin_lock(lock: &chan->pendq_lock);
1027 for (q_num = 0; q_num < chan->q_nr; q_num++) {
1028 list_for_each_entry(iter, &chan->pendq[q_num], node) {
1029 if (iter->ctx->idx == ctx->idx) {
1030 if (count >=
1031 (iter->to_h_blks * VK_MSGQ_BLK_SIZE)) {
1032 list_del(entry: &iter->node);
1033 atomic_dec(v: &ctx->pend_cnt);
1034 entry = iter;
1035 } else {
1036 /* buffer not big enough */
1037 rc = -EMSGSIZE;
1038 }
1039 goto read_loop_exit;
1040 }
1041 }
1042 }
1043read_loop_exit:
1044 spin_unlock(lock: &chan->pendq_lock);
1045
1046 if (entry) {
1047 /* retrieve the passed down msg_id */
1048 set_msg_id(msg: &entry->to_h_msg[0], val: entry->usr_msg_id);
1049 rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE;
1050 if (copy_to_user(to: buf, from: entry->to_h_msg, n: rsp_length) == 0)
1051 rc = rsp_length;
1052
1053 bcm_vk_free_wkent(dev, entry);
1054 } else if (rc == -EMSGSIZE) {
1055 struct vk_msg_blk tmp_msg = entry->to_h_msg[0];
1056
1057 /*
1058 * in this case, return just the first block, so
1059 * that app knows what size it is looking for.
1060 */
1061 set_msg_id(msg: &tmp_msg, val: entry->usr_msg_id);
1062 tmp_msg.size = entry->to_h_blks - 1;
1063 if (copy_to_user(to: buf, from: &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) {
1064 dev_err(dev, "Error return 1st block in -EMSGSIZE\n");
1065 rc = -EFAULT;
1066 }
1067 }
1068 return rc;
1069}
1070
1071ssize_t bcm_vk_write(struct file *p_file,
1072 const char __user *buf,
1073 size_t count,
1074 loff_t *f_pos)
1075{
1076 ssize_t rc;
1077 struct bcm_vk_ctx *ctx = p_file->private_data;
1078 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,
1079 miscdev);
1080 struct bcm_vk_msgq __iomem *msgq;
1081 struct device *dev = &vk->pdev->dev;
1082 struct bcm_vk_wkent *entry;
1083 u32 sgl_extra_blks;
1084 u32 q_num;
1085 u32 msg_size;
1086 u32 msgq_size;
1087
1088 if (!bcm_vk_drv_access_ok(vk))
1089 return -EPERM;
1090
1091 dev_dbg(dev, "Msg count %zu\n", count);
1092
1093 /* first, do sanity check where count should be multiple of basic blk */
1094 if (count & (VK_MSGQ_BLK_SIZE - 1)) {
1095 dev_err(dev, "Failure with size %zu not multiple of %zu\n",
1096 count, VK_MSGQ_BLK_SIZE);
1097 rc = -EINVAL;
1098 goto write_err;
1099 }
1100
1101 /* allocate the work entry + buffer for size count and inband sgl */
1102 entry = kzalloc(size: sizeof(*entry) + count + vk->ib_sgl_size,
1103 GFP_KERNEL);
1104 if (!entry) {
1105 rc = -ENOMEM;
1106 goto write_err;
1107 }
1108
1109 /* now copy msg from user space, and then formulate the work entry */
1110 if (copy_from_user(to: &entry->to_v_msg[0], from: buf, n: count)) {
1111 rc = -EFAULT;
1112 goto write_free_ent;
1113 }
1114
1115 entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT;
1116 entry->ctx = ctx;
1117
1118 /* do a check on the blk size which could not exceed queue space */
1119 q_num = get_q_num(msg: &entry->to_v_msg[0]);
1120 msgq = vk->to_v_msg_chan.msgq[q_num];
1121 msgq_size = readl_relaxed(&msgq->size);
1122 if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT)
1123 > (msgq_size - 1)) {
1124 dev_err(dev, "Blk size %d exceed max queue size allowed %d\n",
1125 entry->to_v_blks, msgq_size - 1);
1126 rc = -EINVAL;
1127 goto write_free_ent;
1128 }
1129
1130 /* Use internal message id */
1131 entry->usr_msg_id = get_msg_id(msg: &entry->to_v_msg[0]);
1132 rc = bcm_vk_get_msg_id(vk);
1133 if (rc == VK_MSG_ID_OVERFLOW) {
1134 dev_err(dev, "msg_id overflow\n");
1135 rc = -EOVERFLOW;
1136 goto write_free_ent;
1137 }
1138 set_msg_id(msg: &entry->to_v_msg[0], val: rc);
1139 ctx->q_num = q_num;
1140
1141 dev_dbg(dev,
1142 "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n",
1143 ctx->q_num, ctx->idx, entry->usr_msg_id,
1144 get_msg_id(&entry->to_v_msg[0]));
1145
1146 if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) {
1147 /* Convert any pointers to sg list */
1148 unsigned int num_planes;
1149 int dir;
1150 struct _vk_data *data;
1151
1152 /*
1153 * check if we are in reset, if so, no buffer transfer is
1154 * allowed and return error.
1155 */
1156 if (vk->reset_pid) {
1157 dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n",
1158 ctx->pid);
1159 rc = -EACCES;
1160 goto write_free_msgid;
1161 }
1162
1163 num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK;
1164 if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD)
1165 dir = DMA_FROM_DEVICE;
1166 else
1167 dir = DMA_TO_DEVICE;
1168
1169 /* Calculate vk_data location */
1170 /* Go to end of the message */
1171 msg_size = entry->to_v_msg[0].size;
1172 if (msg_size > entry->to_v_blks) {
1173 rc = -EMSGSIZE;
1174 goto write_free_msgid;
1175 }
1176
1177 data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1];
1178
1179 /* Now back up to the start of the pointers */
1180 data -= num_planes;
1181
1182 /* Convert user addresses to DMA SG List */
1183 rc = bcm_vk_sg_alloc(dev, dma: entry->dma, dir, vkdata: data, num: num_planes);
1184 if (rc)
1185 goto write_free_msgid;
1186
1187 atomic_inc(v: &ctx->dma_cnt);
1188 /* try to embed inband sgl */
1189 sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data,
1190 num_planes);
1191 entry->to_v_blks += sgl_extra_blks;
1192 entry->to_v_msg[0].size += sgl_extra_blks;
1193 } else if (entry->to_v_msg[0].function_id == VK_FID_INIT &&
1194 entry->to_v_msg[0].context_id == VK_NEW_CTX) {
1195 /*
1196 * Init happens in 2 stages, only the first stage contains the
1197 * pid that needs translating.
1198 */
1199 pid_t org_pid, pid;
1200
1201 /*
1202 * translate the pid into the unique host space as user
1203 * may run sessions inside containers or process
1204 * namespaces.
1205 */
1206#define VK_MSG_PID_MASK 0xffffff00
1207#define VK_MSG_PID_SH 8
1208 org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK)
1209 >> VK_MSG_PID_SH;
1210
1211 pid = task_tgid_nr(current);
1212 entry->to_v_msg[0].arg =
1213 (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) |
1214 (pid << VK_MSG_PID_SH);
1215 if (org_pid != pid)
1216 dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n",
1217 org_pid, org_pid, pid, pid);
1218 }
1219
1220 /*
1221 * store work entry to pending queue until a response is received.
1222 * This needs to be done before enqueuing the message
1223 */
1224 bcm_vk_append_pendq(chan: &vk->to_v_msg_chan, q_num, entry);
1225
1226 rc = bcm_to_v_msg_enqueue(vk, entry);
1227 if (rc) {
1228 dev_err(dev, "Fail to enqueue msg to to_v queue\n");
1229
1230 /* remove message from pending list */
1231 entry = bcm_vk_dequeue_pending
1232 (vk,
1233 chan: &vk->to_v_msg_chan,
1234 q_num,
1235 msg_id: get_msg_id(msg: &entry->to_v_msg[0]));
1236 goto write_free_ent;
1237 }
1238
1239 return count;
1240
1241write_free_msgid:
1242 bcm_vk_msgid_bitmap_clear(vk, start: get_msg_id(msg: &entry->to_v_msg[0]), nbits: 1);
1243write_free_ent:
1244 kfree(objp: entry);
1245write_err:
1246 return rc;
1247}
1248
1249__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait)
1250{
1251 __poll_t ret = 0;
1252 int cnt;
1253 struct bcm_vk_ctx *ctx = p_file->private_data;
1254 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
1255 struct device *dev = &vk->pdev->dev;
1256
1257 poll_wait(filp: p_file, wait_address: &ctx->rd_wq, p: wait);
1258
1259 cnt = atomic_read(v: &ctx->pend_cnt);
1260 if (cnt) {
1261 ret = (__force __poll_t)(POLLIN | POLLRDNORM);
1262 if (cnt < 0) {
1263 dev_err(dev, "Error cnt %d, setting back to 0", cnt);
1264 atomic_set(v: &ctx->pend_cnt, i: 0);
1265 }
1266 }
1267
1268 return ret;
1269}
1270
1271int bcm_vk_release(struct inode *inode, struct file *p_file)
1272{
1273 int ret;
1274 struct bcm_vk_ctx *ctx = p_file->private_data;
1275 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);
1276 struct device *dev = &vk->pdev->dev;
1277 pid_t pid = ctx->pid;
1278 int dma_cnt;
1279 unsigned long timeout, start_time;
1280
1281 /*
1282 * if there are outstanding DMA transactions, need to delay long enough
1283 * to ensure that the card side would have stopped touching the host buffer
1284 * and its SGL list. A race condition could happen if the host app is killed
1285 * abruptly, eg kill -9, while some DMA transfer orders are still inflight.
1286 * Nothing could be done except for a delay as host side is running in a
1287 * completely async fashion.
1288 */
1289 start_time = jiffies;
1290 timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS);
1291 do {
1292 if (time_after(jiffies, timeout)) {
1293 dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n",
1294 dma_cnt, ctx->idx, pid);
1295 break;
1296 }
1297 dma_cnt = atomic_read(v: &ctx->dma_cnt);
1298 cpu_relax();
1299 cond_resched();
1300 } while (dma_cnt);
1301 dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n",
1302 ctx->idx, pid, jiffies_to_msecs(jiffies - start_time));
1303
1304 bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_v_msg_chan, ctx);
1305 bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_h_msg_chan, ctx);
1306
1307 ret = bcm_vk_free_ctx(vk, ctx);
1308 if (ret == 0)
1309 ret = bcm_vk_handle_last_sess(vk, pid, q_num: ctx->q_num);
1310 else
1311 ret = 0;
1312
1313 kref_put(kref: &vk->kref, release: bcm_vk_release_data);
1314
1315 return ret;
1316}
1317
1318int bcm_vk_msg_init(struct bcm_vk *vk)
1319{
1320 struct device *dev = &vk->pdev->dev;
1321 int ret;
1322
1323 if (bcm_vk_data_init(vk)) {
1324 dev_err(dev, "Error initializing internal data structures\n");
1325 return -EINVAL;
1326 }
1327
1328 if (bcm_vk_msg_chan_init(chan: &vk->to_v_msg_chan) ||
1329 bcm_vk_msg_chan_init(chan: &vk->to_h_msg_chan)) {
1330 dev_err(dev, "Error initializing communication channel\n");
1331 return -EIO;
1332 }
1333
1334 /* read msgq info if ready */
1335 ret = bcm_vk_sync_msgq(vk, force_sync: false);
1336 if (ret && (ret != -EAGAIN)) {
1337 dev_err(dev, "Error reading comm msg Q info\n");
1338 return -EIO;
1339 }
1340
1341 return 0;
1342}
1343
1344void bcm_vk_msg_remove(struct bcm_vk *vk)
1345{
1346 bcm_vk_blk_drv_access(vk);
1347
1348 /* drain all pending items */
1349 bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_v_msg_chan, NULL);
1350 bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_h_msg_chan, NULL);
1351}
1352
1353

source code of linux/drivers/misc/bcm-vk/bcm_vk_msg.c