1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright 2018-2020 Broadcom. |
4 | */ |
5 | |
6 | #include <linux/delay.h> |
7 | #include <linux/fs.h> |
8 | #include <linux/hash.h> |
9 | #include <linux/interrupt.h> |
10 | #include <linux/list.h> |
11 | #include <linux/module.h> |
12 | #include <linux/poll.h> |
13 | #include <linux/sizes.h> |
14 | #include <linux/spinlock.h> |
15 | #include <linux/timer.h> |
16 | |
17 | #include "bcm_vk.h" |
18 | #include "bcm_vk_msg.h" |
19 | #include "bcm_vk_sg.h" |
20 | |
21 | /* functions to manipulate the transport id in msg block */ |
22 | #define BCM_VK_MSG_Q_SHIFT 4 |
23 | #define BCM_VK_MSG_Q_MASK 0xF |
24 | #define BCM_VK_MSG_ID_MASK 0xFFF |
25 | |
26 | #define BCM_VK_DMA_DRAIN_MAX_MS 2000 |
27 | |
28 | /* number x q_size will be the max number of msg processed per loop */ |
29 | #define BCM_VK_MSG_PROC_MAX_LOOP 2 |
30 | |
31 | /* module parameter */ |
32 | static bool hb_mon = true; |
33 | module_param(hb_mon, bool, 0444); |
34 | MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n" ); |
35 | static int batch_log = 1; |
36 | module_param(batch_log, int, 0444); |
37 | MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n" ); |
38 | |
39 | static bool hb_mon_is_on(void) |
40 | { |
41 | return hb_mon; |
42 | } |
43 | |
44 | static u32 get_q_num(const struct vk_msg_blk *msg) |
45 | { |
46 | u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK; |
47 | |
48 | if (q_num >= VK_MSGQ_PER_CHAN_MAX) |
49 | q_num = VK_MSGQ_NUM_DEFAULT; |
50 | return q_num; |
51 | } |
52 | |
53 | static void set_q_num(struct vk_msg_blk *msg, u32 q_num) |
54 | { |
55 | u32 trans_q; |
56 | |
57 | if (q_num >= VK_MSGQ_PER_CHAN_MAX) |
58 | trans_q = VK_MSGQ_NUM_DEFAULT; |
59 | else |
60 | trans_q = q_num; |
61 | |
62 | msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q; |
63 | } |
64 | |
65 | static u32 get_msg_id(const struct vk_msg_blk *msg) |
66 | { |
67 | return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK); |
68 | } |
69 | |
70 | static void set_msg_id(struct vk_msg_blk *msg, u32 val) |
71 | { |
72 | msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg); |
73 | } |
74 | |
75 | static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc) |
76 | { |
77 | return ((idx + inc) & qinfo->q_mask); |
78 | } |
79 | |
80 | static |
81 | struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo, |
82 | u32 idx) |
83 | { |
84 | return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx); |
85 | } |
86 | |
87 | static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq, |
88 | const struct bcm_vk_sync_qinfo *qinfo) |
89 | { |
90 | u32 wr_idx, rd_idx; |
91 | |
92 | wr_idx = readl_relaxed(&msgq->wr_idx); |
93 | rd_idx = readl_relaxed(&msgq->rd_idx); |
94 | |
95 | return ((wr_idx - rd_idx) & qinfo->q_mask); |
96 | } |
97 | |
98 | static |
99 | u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, |
100 | const struct bcm_vk_sync_qinfo *qinfo) |
101 | { |
102 | return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1); |
103 | } |
104 | |
105 | /* number of retries when enqueue message fails before returning EAGAIN */ |
106 | #define BCM_VK_H2VK_ENQ_RETRY 10 |
107 | #define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50 |
108 | |
109 | bool bcm_vk_drv_access_ok(struct bcm_vk *vk) |
110 | { |
111 | return (!!atomic_read(v: &vk->msgq_inited)); |
112 | } |
113 | |
114 | void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) |
115 | { |
116 | struct bcm_vk_alert *alert = &vk->host_alert; |
117 | unsigned long flags; |
118 | |
119 | /* use irqsave version as this maybe called inside timer interrupt */ |
120 | spin_lock_irqsave(&vk->host_alert_lock, flags); |
121 | alert->notfs |= bit_mask; |
122 | spin_unlock_irqrestore(lock: &vk->host_alert_lock, flags); |
123 | |
124 | if (test_and_set_bit(nr: BCM_VK_WQ_NOTF_PEND, addr: vk->wq_offload) == 0) |
125 | queue_work(wq: vk->wq_thread, work: &vk->wq_work); |
126 | } |
127 | |
128 | /* |
129 | * Heartbeat related defines |
130 | * The heartbeat from host is a last resort. If stuck condition happens |
131 | * on the card, firmware is supposed to detect it. Therefore, the heartbeat |
132 | * values used will be more relaxed on the driver, which need to be bigger |
133 | * than the watchdog timeout on the card. The watchdog timeout on the card |
134 | * is 20s, with a jitter of 2s => 22s. We use a value of 27s here. |
135 | */ |
136 | #define BCM_VK_HB_TIMER_S 3 |
137 | #define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ) |
138 | #define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S) |
139 | |
140 | static void bcm_vk_hb_poll(struct work_struct *work) |
141 | { |
142 | u32 uptime_s; |
143 | struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl, |
144 | work); |
145 | struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl); |
146 | |
147 | if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) { |
148 | /* read uptime from register and compare */ |
149 | uptime_s = vkread32(vk, bar: BAR_0, BAR_OS_UPTIME); |
150 | |
151 | if (uptime_s == hb->last_uptime) |
152 | hb->lost_cnt++; |
153 | else /* reset to avoid accumulation */ |
154 | hb->lost_cnt = 0; |
155 | |
156 | dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n" , |
157 | hb->last_uptime, uptime_s, hb->lost_cnt); |
158 | |
159 | /* |
160 | * if the interface goes down without any activity, a value |
161 | * of 0xFFFFFFFF will be continuously read, and the detection |
162 | * will be happened eventually. |
163 | */ |
164 | hb->last_uptime = uptime_s; |
165 | } else { |
166 | /* reset heart beat lost cnt */ |
167 | hb->lost_cnt = 0; |
168 | } |
169 | |
170 | /* next, check if heartbeat exceeds limit */ |
171 | if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) { |
172 | dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n" , |
173 | BCM_VK_HB_LOST_MAX, |
174 | BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S); |
175 | |
176 | bcm_vk_blk_drv_access(vk); |
177 | bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL); |
178 | } |
179 | /* re-arm timer */ |
180 | schedule_delayed_work(dwork: &hb->work, BCM_VK_HB_TIMER_VALUE); |
181 | } |
182 | |
183 | void bcm_vk_hb_init(struct bcm_vk *vk) |
184 | { |
185 | struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; |
186 | |
187 | INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll); |
188 | schedule_delayed_work(dwork: &hb->work, BCM_VK_HB_TIMER_VALUE); |
189 | } |
190 | |
191 | void bcm_vk_hb_deinit(struct bcm_vk *vk) |
192 | { |
193 | struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl; |
194 | |
195 | cancel_delayed_work_sync(dwork: &hb->work); |
196 | } |
197 | |
198 | static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk, |
199 | unsigned int start, |
200 | unsigned int nbits) |
201 | { |
202 | spin_lock(lock: &vk->msg_id_lock); |
203 | bitmap_clear(map: vk->bmap, start, nbits); |
204 | spin_unlock(lock: &vk->msg_id_lock); |
205 | } |
206 | |
207 | /* |
208 | * allocate a ctx per file struct |
209 | */ |
210 | static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid) |
211 | { |
212 | u32 i; |
213 | struct bcm_vk_ctx *ctx = NULL; |
214 | u32 hash_idx = hash_32(val: pid, VK_PID_HT_SHIFT_BIT); |
215 | |
216 | spin_lock(lock: &vk->ctx_lock); |
217 | |
218 | /* check if it is in reset, if so, don't allow */ |
219 | if (vk->reset_pid) { |
220 | dev_err(&vk->pdev->dev, |
221 | "No context allowed during reset by pid %d\n" , |
222 | vk->reset_pid); |
223 | |
224 | goto in_reset_exit; |
225 | } |
226 | |
227 | for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { |
228 | if (!vk->ctx[i].in_use) { |
229 | vk->ctx[i].in_use = true; |
230 | ctx = &vk->ctx[i]; |
231 | break; |
232 | } |
233 | } |
234 | |
235 | if (!ctx) { |
236 | dev_err(&vk->pdev->dev, "All context in use\n" ); |
237 | |
238 | goto all_in_use_exit; |
239 | } |
240 | |
241 | /* set the pid and insert it to hash table */ |
242 | ctx->pid = pid; |
243 | ctx->hash_idx = hash_idx; |
244 | list_add_tail(new: &ctx->node, head: &vk->pid_ht[hash_idx].head); |
245 | |
246 | /* increase kref */ |
247 | kref_get(kref: &vk->kref); |
248 | |
249 | /* clear counter */ |
250 | atomic_set(v: &ctx->pend_cnt, i: 0); |
251 | atomic_set(v: &ctx->dma_cnt, i: 0); |
252 | init_waitqueue_head(&ctx->rd_wq); |
253 | |
254 | all_in_use_exit: |
255 | in_reset_exit: |
256 | spin_unlock(lock: &vk->ctx_lock); |
257 | |
258 | return ctx; |
259 | } |
260 | |
261 | static u16 bcm_vk_get_msg_id(struct bcm_vk *vk) |
262 | { |
263 | u16 rc = VK_MSG_ID_OVERFLOW; |
264 | u16 test_bit_count = 0; |
265 | |
266 | spin_lock(lock: &vk->msg_id_lock); |
267 | while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) { |
268 | /* |
269 | * first time come in this loop, msg_id will be 0 |
270 | * and the first one tested will be 1. We skip |
271 | * VK_SIMPLEX_MSG_ID (0) for one way host2vk |
272 | * communication |
273 | */ |
274 | vk->msg_id++; |
275 | if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE) |
276 | vk->msg_id = 1; |
277 | |
278 | if (test_bit(vk->msg_id, vk->bmap)) { |
279 | test_bit_count++; |
280 | continue; |
281 | } |
282 | rc = vk->msg_id; |
283 | bitmap_set(map: vk->bmap, start: vk->msg_id, nbits: 1); |
284 | break; |
285 | } |
286 | spin_unlock(lock: &vk->msg_id_lock); |
287 | |
288 | return rc; |
289 | } |
290 | |
291 | static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx) |
292 | { |
293 | u32 idx; |
294 | u32 hash_idx; |
295 | pid_t pid; |
296 | struct bcm_vk_ctx *entry; |
297 | int count = 0; |
298 | |
299 | if (!ctx) { |
300 | dev_err(&vk->pdev->dev, "NULL context detected\n" ); |
301 | return -EINVAL; |
302 | } |
303 | idx = ctx->idx; |
304 | pid = ctx->pid; |
305 | |
306 | spin_lock(lock: &vk->ctx_lock); |
307 | |
308 | if (!vk->ctx[idx].in_use) { |
309 | dev_err(&vk->pdev->dev, "context[%d] not in use!\n" , idx); |
310 | } else { |
311 | vk->ctx[idx].in_use = false; |
312 | vk->ctx[idx].miscdev = NULL; |
313 | |
314 | /* Remove it from hash list and see if it is the last one. */ |
315 | list_del(entry: &ctx->node); |
316 | hash_idx = ctx->hash_idx; |
317 | list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) { |
318 | if (entry->pid == pid) |
319 | count++; |
320 | } |
321 | } |
322 | |
323 | spin_unlock(lock: &vk->ctx_lock); |
324 | |
325 | return count; |
326 | } |
327 | |
328 | static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry) |
329 | { |
330 | int proc_cnt; |
331 | |
332 | bcm_vk_sg_free(dev, dma: entry->dma, VK_DMA_MAX_ADDRS, proc_cnt: &proc_cnt); |
333 | if (proc_cnt) |
334 | atomic_dec(v: &entry->ctx->dma_cnt); |
335 | |
336 | kfree(objp: entry->to_h_msg); |
337 | kfree(objp: entry); |
338 | } |
339 | |
340 | static void bcm_vk_drain_all_pend(struct device *dev, |
341 | struct bcm_vk_msg_chan *chan, |
342 | struct bcm_vk_ctx *ctx) |
343 | { |
344 | u32 num; |
345 | struct bcm_vk_wkent *entry, *tmp; |
346 | struct bcm_vk *vk; |
347 | struct list_head del_q; |
348 | |
349 | if (ctx) |
350 | vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); |
351 | |
352 | INIT_LIST_HEAD(list: &del_q); |
353 | spin_lock(lock: &chan->pendq_lock); |
354 | for (num = 0; num < chan->q_nr; num++) { |
355 | list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) { |
356 | if ((!ctx) || (entry->ctx->idx == ctx->idx)) { |
357 | list_move_tail(list: &entry->node, head: &del_q); |
358 | } |
359 | } |
360 | } |
361 | spin_unlock(lock: &chan->pendq_lock); |
362 | |
363 | /* batch clean up */ |
364 | num = 0; |
365 | list_for_each_entry_safe(entry, tmp, &del_q, node) { |
366 | list_del(entry: &entry->node); |
367 | num++; |
368 | if (ctx) { |
369 | struct vk_msg_blk *msg; |
370 | int bit_set; |
371 | bool responded; |
372 | u32 msg_id; |
373 | |
374 | /* if it is specific ctx, log for any stuck */ |
375 | msg = entry->to_v_msg; |
376 | msg_id = get_msg_id(msg); |
377 | bit_set = test_bit(msg_id, vk->bmap); |
378 | responded = entry->to_h_msg ? true : false; |
379 | if (num <= batch_log) |
380 | dev_info(dev, |
381 | "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n" , |
382 | msg->function_id, msg->size, |
383 | msg_id, entry->seq_num, |
384 | msg->context_id, entry->ctx->idx, |
385 | msg->cmd, msg->arg, |
386 | responded ? "T" : "F" , bit_set); |
387 | if (responded) |
388 | atomic_dec(v: &ctx->pend_cnt); |
389 | else if (bit_set) |
390 | bcm_vk_msgid_bitmap_clear(vk, start: msg_id, nbits: 1); |
391 | } |
392 | bcm_vk_free_wkent(dev, entry); |
393 | } |
394 | if (num && ctx) |
395 | dev_info(dev, "Total drained items %d [fd-%d]\n" , |
396 | num, ctx->idx); |
397 | } |
398 | |
399 | void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk) |
400 | { |
401 | bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_v_msg_chan, NULL); |
402 | bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_h_msg_chan, NULL); |
403 | } |
404 | |
405 | /* |
406 | * Function to sync up the messages queue info that is provided by BAR1 |
407 | */ |
408 | int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) |
409 | { |
410 | struct bcm_vk_msgq __iomem *msgq; |
411 | struct device *dev = &vk->pdev->dev; |
412 | u32 msgq_off; |
413 | u32 num_q; |
414 | struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan, |
415 | &vk->to_h_msg_chan}; |
416 | struct bcm_vk_msg_chan *chan; |
417 | int i, j; |
418 | int ret = 0; |
419 | |
420 | /* |
421 | * If the driver is loaded at startup where vk OS is not up yet, |
422 | * the msgq-info may not be available until a later time. In |
423 | * this case, we skip and the sync function is supposed to be |
424 | * called again. |
425 | */ |
426 | if (!bcm_vk_msgq_marker_valid(vk)) { |
427 | dev_info(dev, "BAR1 msgq marker not initialized.\n" ); |
428 | return -EAGAIN; |
429 | } |
430 | |
431 | msgq_off = vkread32(vk, bar: BAR_1, VK_BAR1_MSGQ_CTRL_OFF); |
432 | |
433 | /* each side is always half the total */ |
434 | num_q = vkread32(vk, bar: BAR_1, VK_BAR1_MSGQ_NR) / 2; |
435 | if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) { |
436 | dev_err(dev, |
437 | "Advertised msgq %d error - max %d allowed\n" , |
438 | num_q, VK_MSGQ_PER_CHAN_MAX); |
439 | return -EINVAL; |
440 | } |
441 | |
442 | vk->to_v_msg_chan.q_nr = num_q; |
443 | vk->to_h_msg_chan.q_nr = num_q; |
444 | |
445 | /* first msgq location */ |
446 | msgq = vk->bar[BAR_1] + msgq_off; |
447 | |
448 | /* |
449 | * if this function is called when it is already inited, |
450 | * something is wrong |
451 | */ |
452 | if (bcm_vk_drv_access_ok(vk) && !force_sync) { |
453 | dev_err(dev, "Msgq info already in sync\n" ); |
454 | return -EPERM; |
455 | } |
456 | |
457 | for (i = 0; i < ARRAY_SIZE(chan_list); i++) { |
458 | chan = chan_list[i]; |
459 | memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo)); |
460 | |
461 | for (j = 0; j < num_q; j++) { |
462 | struct bcm_vk_sync_qinfo *qinfo; |
463 | u32 msgq_start; |
464 | u32 msgq_size; |
465 | u32 msgq_nxt; |
466 | u32 msgq_db_offset, q_db_offset; |
467 | |
468 | chan->msgq[j] = msgq; |
469 | msgq_start = readl_relaxed(&msgq->start); |
470 | msgq_size = readl_relaxed(&msgq->size); |
471 | msgq_nxt = readl_relaxed(&msgq->nxt); |
472 | msgq_db_offset = readl_relaxed(&msgq->db_offset); |
473 | q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1)); |
474 | if (q_db_offset == (~msgq_db_offset >> DB_SHIFT)) |
475 | msgq_db_offset = q_db_offset; |
476 | else |
477 | /* fall back to default */ |
478 | msgq_db_offset = VK_BAR0_Q_DB_BASE(j); |
479 | |
480 | dev_info(dev, |
481 | "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n" , |
482 | j, |
483 | readw_relaxed(&msgq->type), |
484 | readw_relaxed(&msgq->num), |
485 | msgq_start, |
486 | msgq_db_offset, |
487 | readl_relaxed(&msgq->rd_idx), |
488 | readl_relaxed(&msgq->wr_idx), |
489 | msgq_size, |
490 | msgq_nxt); |
491 | |
492 | qinfo = &chan->sync_qinfo[j]; |
493 | /* formulate and record static info */ |
494 | qinfo->q_start = vk->bar[BAR_1] + msgq_start; |
495 | qinfo->q_size = msgq_size; |
496 | /* set low threshold as 50% or 1/2 */ |
497 | qinfo->q_low = qinfo->q_size >> 1; |
498 | qinfo->q_mask = qinfo->q_size - 1; |
499 | qinfo->q_db_offset = msgq_db_offset; |
500 | |
501 | msgq++; |
502 | } |
503 | } |
504 | atomic_set(v: &vk->msgq_inited, i: 1); |
505 | |
506 | return ret; |
507 | } |
508 | |
509 | static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan) |
510 | { |
511 | u32 i; |
512 | |
513 | mutex_init(&chan->msgq_mutex); |
514 | spin_lock_init(&chan->pendq_lock); |
515 | for (i = 0; i < VK_MSGQ_MAX_NR; i++) |
516 | INIT_LIST_HEAD(list: &chan->pendq[i]); |
517 | |
518 | return 0; |
519 | } |
520 | |
521 | static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num, |
522 | struct bcm_vk_wkent *entry) |
523 | { |
524 | struct bcm_vk_ctx *ctx; |
525 | |
526 | spin_lock(lock: &chan->pendq_lock); |
527 | list_add_tail(new: &entry->node, head: &chan->pendq[q_num]); |
528 | if (entry->to_h_msg) { |
529 | ctx = entry->ctx; |
530 | atomic_inc(v: &ctx->pend_cnt); |
531 | wake_up_interruptible(&ctx->rd_wq); |
532 | } |
533 | spin_unlock(lock: &chan->pendq_lock); |
534 | } |
535 | |
536 | static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk, |
537 | struct bcm_vk_wkent *entry, |
538 | struct _vk_data *data, |
539 | unsigned int num_planes) |
540 | { |
541 | unsigned int i; |
542 | unsigned int item_cnt = 0; |
543 | struct device *dev = &vk->pdev->dev; |
544 | struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; |
545 | struct vk_msg_blk *msg = &entry->to_v_msg[0]; |
546 | struct bcm_vk_msgq __iomem *msgq; |
547 | struct bcm_vk_sync_qinfo *qinfo; |
548 | u32 ib_sgl_size = 0; |
549 | u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks]; |
550 | u32 avail; |
551 | u32 q_num; |
552 | |
553 | /* check if high watermark is hit, and if so, skip */ |
554 | q_num = get_q_num(msg); |
555 | msgq = chan->msgq[q_num]; |
556 | qinfo = &chan->sync_qinfo[q_num]; |
557 | avail = msgq_avail_space(msgq, qinfo); |
558 | if (avail < qinfo->q_low) { |
559 | dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n" , |
560 | avail, qinfo->q_size); |
561 | return 0; |
562 | } |
563 | |
564 | for (i = 0; i < num_planes; i++) { |
565 | if (data[i].address && |
566 | (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) { |
567 | item_cnt++; |
568 | memcpy(buf, entry->dma[i].sglist, data[i].size); |
569 | ib_sgl_size += data[i].size; |
570 | buf += data[i].size; |
571 | } |
572 | } |
573 | |
574 | dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n" , |
575 | item_cnt, ib_sgl_size, vk->ib_sgl_size); |
576 | |
577 | /* round up size */ |
578 | ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1) |
579 | >> VK_MSGQ_BLK_SZ_SHIFT; |
580 | |
581 | return ib_sgl_size; |
582 | } |
583 | |
584 | void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val) |
585 | { |
586 | struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; |
587 | struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num]; |
588 | |
589 | vkwrite32(vk, value: db_val, bar: BAR_0, offset: qinfo->q_db_offset); |
590 | } |
591 | |
592 | static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry) |
593 | { |
594 | static u32 seq_num; |
595 | struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan; |
596 | struct device *dev = &vk->pdev->dev; |
597 | struct vk_msg_blk *src = &entry->to_v_msg[0]; |
598 | |
599 | struct vk_msg_blk __iomem *dst; |
600 | struct bcm_vk_msgq __iomem *msgq; |
601 | struct bcm_vk_sync_qinfo *qinfo; |
602 | u32 q_num = get_q_num(msg: src); |
603 | u32 wr_idx; /* local copy */ |
604 | u32 i; |
605 | u32 avail; |
606 | u32 retry; |
607 | |
608 | if (entry->to_v_blks != src->size + 1) { |
609 | dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n" , |
610 | entry->to_v_blks, |
611 | src->size + 1, |
612 | get_msg_id(src), |
613 | src->function_id, |
614 | src->context_id); |
615 | return -EMSGSIZE; |
616 | } |
617 | |
618 | msgq = chan->msgq[q_num]; |
619 | qinfo = &chan->sync_qinfo[q_num]; |
620 | |
621 | mutex_lock(&chan->msgq_mutex); |
622 | |
623 | avail = msgq_avail_space(msgq, qinfo); |
624 | |
625 | /* if not enough space, return EAGAIN and let app handles it */ |
626 | retry = 0; |
627 | while ((avail < entry->to_v_blks) && |
628 | (retry++ < BCM_VK_H2VK_ENQ_RETRY)) { |
629 | mutex_unlock(lock: &chan->msgq_mutex); |
630 | |
631 | msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS); |
632 | mutex_lock(&chan->msgq_mutex); |
633 | avail = msgq_avail_space(msgq, qinfo); |
634 | } |
635 | if (retry > BCM_VK_H2VK_ENQ_RETRY) { |
636 | mutex_unlock(lock: &chan->msgq_mutex); |
637 | return -EAGAIN; |
638 | } |
639 | |
640 | /* at this point, mutex is taken and there is enough space */ |
641 | entry->seq_num = seq_num++; /* update debug seq number */ |
642 | wr_idx = readl_relaxed(&msgq->wr_idx); |
643 | |
644 | if (wr_idx >= qinfo->q_size) { |
645 | dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!" , |
646 | wr_idx, qinfo->q_size); |
647 | bcm_vk_blk_drv_access(vk); |
648 | bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN); |
649 | goto idx_err; |
650 | } |
651 | |
652 | dst = msgq_blk_addr(qinfo, idx: wr_idx); |
653 | for (i = 0; i < entry->to_v_blks; i++) { |
654 | memcpy_toio(dst, src, sizeof(*dst)); |
655 | |
656 | src++; |
657 | wr_idx = msgq_inc(qinfo, idx: wr_idx, inc: 1); |
658 | dst = msgq_blk_addr(qinfo, idx: wr_idx); |
659 | } |
660 | |
661 | /* flush the write pointer */ |
662 | writel(val: wr_idx, addr: &msgq->wr_idx); |
663 | |
664 | /* log new info for debugging */ |
665 | dev_dbg(dev, |
666 | "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n" , |
667 | readl_relaxed(&msgq->num), |
668 | readl_relaxed(&msgq->rd_idx), |
669 | wr_idx, |
670 | entry->to_v_blks, |
671 | msgq_occupied(msgq, qinfo), |
672 | msgq_avail_space(msgq, qinfo), |
673 | readl_relaxed(&msgq->size)); |
674 | /* |
675 | * press door bell based on queue number. 1 is added to the wr_idx |
676 | * to avoid the value of 0 appearing on the VK side to distinguish |
677 | * from initial value. |
678 | */ |
679 | bcm_to_v_q_doorbell(vk, q_num, db_val: wr_idx + 1); |
680 | idx_err: |
681 | mutex_unlock(lock: &chan->msgq_mutex); |
682 | return 0; |
683 | } |
684 | |
685 | int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type, |
686 | const pid_t pid, const u32 q_num) |
687 | { |
688 | int rc = 0; |
689 | struct bcm_vk_wkent *entry; |
690 | struct device *dev = &vk->pdev->dev; |
691 | |
692 | /* |
693 | * check if the marker is still good. Sometimes, the PCIe interface may |
694 | * have gone done, and if so and we ship down thing based on broken |
695 | * values, kernel may panic. |
696 | */ |
697 | if (!bcm_vk_msgq_marker_valid(vk)) { |
698 | dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n" , |
699 | vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY)); |
700 | return -EINVAL; |
701 | } |
702 | |
703 | entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL); |
704 | if (!entry) |
705 | return -ENOMEM; |
706 | entry->to_v_blks = 1; /* always 1 block */ |
707 | |
708 | /* fill up necessary data */ |
709 | entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN; |
710 | set_q_num(msg: &entry->to_v_msg[0], q_num); |
711 | set_msg_id(msg: &entry->to_v_msg[0], VK_SIMPLEX_MSG_ID); |
712 | |
713 | entry->to_v_msg[0].cmd = shut_type; |
714 | entry->to_v_msg[0].arg = pid; |
715 | |
716 | rc = bcm_to_v_msg_enqueue(vk, entry); |
717 | if (rc) |
718 | dev_err(dev, |
719 | "Sending shutdown message to q %d for pid %d fails.\n" , |
720 | get_q_num(&entry->to_v_msg[0]), pid); |
721 | |
722 | kfree(objp: entry); |
723 | |
724 | return rc; |
725 | } |
726 | |
727 | static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid, |
728 | const u32 q_num) |
729 | { |
730 | int rc = 0; |
731 | struct device *dev = &vk->pdev->dev; |
732 | |
733 | /* |
734 | * don't send down or do anything if message queue is not initialized |
735 | * and if it is the reset session, clear it. |
736 | */ |
737 | if (!bcm_vk_drv_access_ok(vk)) { |
738 | if (vk->reset_pid == pid) |
739 | vk->reset_pid = 0; |
740 | return -EPERM; |
741 | } |
742 | |
743 | dev_dbg(dev, "No more sessions, shut down pid %d\n" , pid); |
744 | |
745 | /* only need to do it if it is not the reset process */ |
746 | if (vk->reset_pid != pid) |
747 | rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num); |
748 | else |
749 | /* put reset_pid to 0 if it is exiting last session */ |
750 | vk->reset_pid = 0; |
751 | |
752 | return rc; |
753 | } |
754 | |
755 | static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk, |
756 | struct bcm_vk_msg_chan *chan, |
757 | u16 q_num, |
758 | u16 msg_id) |
759 | { |
760 | struct bcm_vk_wkent *entry = NULL, *iter; |
761 | |
762 | spin_lock(lock: &chan->pendq_lock); |
763 | list_for_each_entry(iter, &chan->pendq[q_num], node) { |
764 | if (get_msg_id(msg: &iter->to_v_msg[0]) == msg_id) { |
765 | list_del(entry: &iter->node); |
766 | entry = iter; |
767 | bcm_vk_msgid_bitmap_clear(vk, start: msg_id, nbits: 1); |
768 | break; |
769 | } |
770 | } |
771 | spin_unlock(lock: &chan->pendq_lock); |
772 | return entry; |
773 | } |
774 | |
775 | s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk) |
776 | { |
777 | struct device *dev = &vk->pdev->dev; |
778 | struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; |
779 | struct vk_msg_blk *data; |
780 | struct vk_msg_blk __iomem *src; |
781 | struct vk_msg_blk *dst; |
782 | struct bcm_vk_msgq __iomem *msgq; |
783 | struct bcm_vk_sync_qinfo *qinfo; |
784 | struct bcm_vk_wkent *entry; |
785 | u32 rd_idx, wr_idx; |
786 | u32 q_num, msg_id, j; |
787 | u32 num_blks; |
788 | s32 total = 0; |
789 | int cnt = 0; |
790 | int msg_processed = 0; |
791 | int max_msg_to_process; |
792 | bool exit_loop; |
793 | |
794 | /* |
795 | * drain all the messages from the queues, and find its pending |
796 | * entry in the to_v queue, based on msg_id & q_num, and move the |
797 | * entry to the to_h pending queue, waiting for user space |
798 | * program to extract |
799 | */ |
800 | mutex_lock(&chan->msgq_mutex); |
801 | |
802 | for (q_num = 0; q_num < chan->q_nr; q_num++) { |
803 | msgq = chan->msgq[q_num]; |
804 | qinfo = &chan->sync_qinfo[q_num]; |
805 | max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size; |
806 | |
807 | rd_idx = readl_relaxed(&msgq->rd_idx); |
808 | wr_idx = readl_relaxed(&msgq->wr_idx); |
809 | msg_processed = 0; |
810 | exit_loop = false; |
811 | while ((rd_idx != wr_idx) && !exit_loop) { |
812 | u8 src_size; |
813 | |
814 | /* |
815 | * Make a local copy and get pointer to src blk |
816 | * The rd_idx is masked before getting the pointer to |
817 | * avoid out of bound access in case the interface goes |
818 | * down. It will end up pointing to the last block in |
819 | * the buffer, but subsequent src->size check would be |
820 | * able to catch this. |
821 | */ |
822 | src = msgq_blk_addr(qinfo, idx: rd_idx & qinfo->q_mask); |
823 | src_size = readb(addr: &src->size); |
824 | |
825 | if ((rd_idx >= qinfo->q_size) || |
826 | (src_size > (qinfo->q_size - 1))) { |
827 | dev_crit(dev, |
828 | "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!" , |
829 | rd_idx, src_size, qinfo->q_size); |
830 | bcm_vk_blk_drv_access(vk); |
831 | bcm_vk_set_host_alert(vk, |
832 | ERR_LOG_HOST_PCIE_DWN); |
833 | goto idx_err; |
834 | } |
835 | |
836 | num_blks = src_size + 1; |
837 | data = kzalloc(size: num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL); |
838 | if (data) { |
839 | /* copy messages and linearize it */ |
840 | dst = data; |
841 | for (j = 0; j < num_blks; j++) { |
842 | memcpy_fromio(dst, src, sizeof(*dst)); |
843 | |
844 | dst++; |
845 | rd_idx = msgq_inc(qinfo, idx: rd_idx, inc: 1); |
846 | src = msgq_blk_addr(qinfo, idx: rd_idx); |
847 | } |
848 | total++; |
849 | } else { |
850 | /* |
851 | * if we could not allocate memory in kernel, |
852 | * that is fatal. |
853 | */ |
854 | dev_crit(dev, "Kernel mem allocation failure.\n" ); |
855 | total = -ENOMEM; |
856 | goto idx_err; |
857 | } |
858 | |
859 | /* flush rd pointer after a message is dequeued */ |
860 | writel(val: rd_idx, addr: &msgq->rd_idx); |
861 | |
862 | /* log new info for debugging */ |
863 | dev_dbg(dev, |
864 | "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n" , |
865 | readl_relaxed(&msgq->num), |
866 | rd_idx, |
867 | wr_idx, |
868 | num_blks, |
869 | msgq_occupied(msgq, qinfo), |
870 | msgq_avail_space(msgq, qinfo), |
871 | readl_relaxed(&msgq->size)); |
872 | |
873 | /* |
874 | * No need to search if it is an autonomous one-way |
875 | * message from driver, as these messages do not bear |
876 | * a to_v pending item. Currently, only the shutdown |
877 | * message falls into this category. |
878 | */ |
879 | if (data->function_id == VK_FID_SHUTDOWN) { |
880 | kfree(objp: data); |
881 | continue; |
882 | } |
883 | |
884 | msg_id = get_msg_id(msg: data); |
885 | /* lookup original message in to_v direction */ |
886 | entry = bcm_vk_dequeue_pending(vk, |
887 | chan: &vk->to_v_msg_chan, |
888 | q_num, |
889 | msg_id); |
890 | |
891 | /* |
892 | * if there is message to does not have prior send, |
893 | * this is the location to add here |
894 | */ |
895 | if (entry) { |
896 | entry->to_h_blks = num_blks; |
897 | entry->to_h_msg = data; |
898 | bcm_vk_append_pendq(chan: &vk->to_h_msg_chan, |
899 | q_num, entry); |
900 | |
901 | } else { |
902 | if (cnt++ < batch_log) |
903 | dev_info(dev, |
904 | "Could not find MsgId[0x%x] for resp func %d bmap %d\n" , |
905 | msg_id, data->function_id, |
906 | test_bit(msg_id, vk->bmap)); |
907 | kfree(objp: data); |
908 | } |
909 | /* Fetch wr_idx to handle more back-to-back events */ |
910 | wr_idx = readl(addr: &msgq->wr_idx); |
911 | |
912 | /* |
913 | * cap the max so that even we try to handle more back-to-back events, |
914 | * so that it won't hold CPU too long or in case rd/wr idexes are |
915 | * corrupted which triggers infinite looping. |
916 | */ |
917 | if (++msg_processed >= max_msg_to_process) { |
918 | dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n" , |
919 | q_num, max_msg_to_process); |
920 | exit_loop = true; |
921 | } |
922 | } |
923 | } |
924 | idx_err: |
925 | mutex_unlock(lock: &chan->msgq_mutex); |
926 | dev_dbg(dev, "total %d drained from queues\n" , total); |
927 | |
928 | return total; |
929 | } |
930 | |
931 | /* |
932 | * init routine for all required data structures |
933 | */ |
934 | static int bcm_vk_data_init(struct bcm_vk *vk) |
935 | { |
936 | int i; |
937 | |
938 | spin_lock_init(&vk->ctx_lock); |
939 | for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) { |
940 | vk->ctx[i].in_use = false; |
941 | vk->ctx[i].idx = i; /* self identity */ |
942 | vk->ctx[i].miscdev = NULL; |
943 | } |
944 | spin_lock_init(&vk->msg_id_lock); |
945 | spin_lock_init(&vk->host_alert_lock); |
946 | vk->msg_id = 0; |
947 | |
948 | /* initialize hash table */ |
949 | for (i = 0; i < VK_PID_HT_SZ; i++) |
950 | INIT_LIST_HEAD(list: &vk->pid_ht[i].head); |
951 | |
952 | return 0; |
953 | } |
954 | |
955 | irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id) |
956 | { |
957 | struct bcm_vk *vk = dev_id; |
958 | |
959 | if (!bcm_vk_drv_access_ok(vk)) { |
960 | dev_err(&vk->pdev->dev, |
961 | "Interrupt %d received when msgq not inited\n" , irq); |
962 | goto skip_schedule_work; |
963 | } |
964 | |
965 | queue_work(wq: vk->wq_thread, work: &vk->wq_work); |
966 | |
967 | skip_schedule_work: |
968 | return IRQ_HANDLED; |
969 | } |
970 | |
971 | int bcm_vk_open(struct inode *inode, struct file *p_file) |
972 | { |
973 | struct bcm_vk_ctx *ctx; |
974 | struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data; |
975 | struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev); |
976 | struct device *dev = &vk->pdev->dev; |
977 | int rc = 0; |
978 | |
979 | /* get a context and set it up for file */ |
980 | ctx = bcm_vk_get_ctx(vk, pid: task_tgid_nr(current)); |
981 | if (!ctx) { |
982 | dev_err(dev, "Error allocating context\n" ); |
983 | rc = -ENOMEM; |
984 | } else { |
985 | /* |
986 | * set up context and replace private data with context for |
987 | * other methods to use. Reason for the context is because |
988 | * it is allowed for multiple sessions to open the sysfs, and |
989 | * for each file open, when upper layer query the response, |
990 | * only those that are tied to a specific open should be |
991 | * returned. The context->idx will be used for such binding |
992 | */ |
993 | ctx->miscdev = miscdev; |
994 | p_file->private_data = ctx; |
995 | dev_dbg(dev, "ctx_returned with idx %d, pid %d\n" , |
996 | ctx->idx, ctx->pid); |
997 | } |
998 | return rc; |
999 | } |
1000 | |
1001 | ssize_t bcm_vk_read(struct file *p_file, |
1002 | char __user *buf, |
1003 | size_t count, |
1004 | loff_t *f_pos) |
1005 | { |
1006 | ssize_t rc = -ENOMSG; |
1007 | struct bcm_vk_ctx *ctx = p_file->private_data; |
1008 | struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, |
1009 | miscdev); |
1010 | struct device *dev = &vk->pdev->dev; |
1011 | struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan; |
1012 | struct bcm_vk_wkent *entry = NULL, *iter; |
1013 | u32 q_num; |
1014 | u32 rsp_length; |
1015 | |
1016 | if (!bcm_vk_drv_access_ok(vk)) |
1017 | return -EPERM; |
1018 | |
1019 | dev_dbg(dev, "Buf count %zu\n" , count); |
1020 | |
1021 | /* |
1022 | * search through the pendq on the to_h chan, and return only those |
1023 | * that belongs to the same context. Search is always from the high to |
1024 | * the low priority queues |
1025 | */ |
1026 | spin_lock(lock: &chan->pendq_lock); |
1027 | for (q_num = 0; q_num < chan->q_nr; q_num++) { |
1028 | list_for_each_entry(iter, &chan->pendq[q_num], node) { |
1029 | if (iter->ctx->idx == ctx->idx) { |
1030 | if (count >= |
1031 | (iter->to_h_blks * VK_MSGQ_BLK_SIZE)) { |
1032 | list_del(entry: &iter->node); |
1033 | atomic_dec(v: &ctx->pend_cnt); |
1034 | entry = iter; |
1035 | } else { |
1036 | /* buffer not big enough */ |
1037 | rc = -EMSGSIZE; |
1038 | } |
1039 | goto read_loop_exit; |
1040 | } |
1041 | } |
1042 | } |
1043 | read_loop_exit: |
1044 | spin_unlock(lock: &chan->pendq_lock); |
1045 | |
1046 | if (entry) { |
1047 | /* retrieve the passed down msg_id */ |
1048 | set_msg_id(msg: &entry->to_h_msg[0], val: entry->usr_msg_id); |
1049 | rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE; |
1050 | if (copy_to_user(to: buf, from: entry->to_h_msg, n: rsp_length) == 0) |
1051 | rc = rsp_length; |
1052 | |
1053 | bcm_vk_free_wkent(dev, entry); |
1054 | } else if (rc == -EMSGSIZE) { |
1055 | struct vk_msg_blk tmp_msg = entry->to_h_msg[0]; |
1056 | |
1057 | /* |
1058 | * in this case, return just the first block, so |
1059 | * that app knows what size it is looking for. |
1060 | */ |
1061 | set_msg_id(msg: &tmp_msg, val: entry->usr_msg_id); |
1062 | tmp_msg.size = entry->to_h_blks - 1; |
1063 | if (copy_to_user(to: buf, from: &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) { |
1064 | dev_err(dev, "Error return 1st block in -EMSGSIZE\n" ); |
1065 | rc = -EFAULT; |
1066 | } |
1067 | } |
1068 | return rc; |
1069 | } |
1070 | |
1071 | ssize_t bcm_vk_write(struct file *p_file, |
1072 | const char __user *buf, |
1073 | size_t count, |
1074 | loff_t *f_pos) |
1075 | { |
1076 | ssize_t rc; |
1077 | struct bcm_vk_ctx *ctx = p_file->private_data; |
1078 | struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, |
1079 | miscdev); |
1080 | struct bcm_vk_msgq __iomem *msgq; |
1081 | struct device *dev = &vk->pdev->dev; |
1082 | struct bcm_vk_wkent *entry; |
1083 | u32 ; |
1084 | u32 q_num; |
1085 | u32 msg_size; |
1086 | u32 msgq_size; |
1087 | |
1088 | if (!bcm_vk_drv_access_ok(vk)) |
1089 | return -EPERM; |
1090 | |
1091 | dev_dbg(dev, "Msg count %zu\n" , count); |
1092 | |
1093 | /* first, do sanity check where count should be multiple of basic blk */ |
1094 | if (count & (VK_MSGQ_BLK_SIZE - 1)) { |
1095 | dev_err(dev, "Failure with size %zu not multiple of %zu\n" , |
1096 | count, VK_MSGQ_BLK_SIZE); |
1097 | rc = -EINVAL; |
1098 | goto write_err; |
1099 | } |
1100 | |
1101 | /* allocate the work entry + buffer for size count and inband sgl */ |
1102 | entry = kzalloc(size: sizeof(*entry) + count + vk->ib_sgl_size, |
1103 | GFP_KERNEL); |
1104 | if (!entry) { |
1105 | rc = -ENOMEM; |
1106 | goto write_err; |
1107 | } |
1108 | |
1109 | /* now copy msg from user space, and then formulate the work entry */ |
1110 | if (copy_from_user(to: &entry->to_v_msg[0], from: buf, n: count)) { |
1111 | rc = -EFAULT; |
1112 | goto write_free_ent; |
1113 | } |
1114 | |
1115 | entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT; |
1116 | entry->ctx = ctx; |
1117 | |
1118 | /* do a check on the blk size which could not exceed queue space */ |
1119 | q_num = get_q_num(msg: &entry->to_v_msg[0]); |
1120 | msgq = vk->to_v_msg_chan.msgq[q_num]; |
1121 | msgq_size = readl_relaxed(&msgq->size); |
1122 | if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT) |
1123 | > (msgq_size - 1)) { |
1124 | dev_err(dev, "Blk size %d exceed max queue size allowed %d\n" , |
1125 | entry->to_v_blks, msgq_size - 1); |
1126 | rc = -EINVAL; |
1127 | goto write_free_ent; |
1128 | } |
1129 | |
1130 | /* Use internal message id */ |
1131 | entry->usr_msg_id = get_msg_id(msg: &entry->to_v_msg[0]); |
1132 | rc = bcm_vk_get_msg_id(vk); |
1133 | if (rc == VK_MSG_ID_OVERFLOW) { |
1134 | dev_err(dev, "msg_id overflow\n" ); |
1135 | rc = -EOVERFLOW; |
1136 | goto write_free_ent; |
1137 | } |
1138 | set_msg_id(msg: &entry->to_v_msg[0], val: rc); |
1139 | ctx->q_num = q_num; |
1140 | |
1141 | dev_dbg(dev, |
1142 | "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n" , |
1143 | ctx->q_num, ctx->idx, entry->usr_msg_id, |
1144 | get_msg_id(&entry->to_v_msg[0])); |
1145 | |
1146 | if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) { |
1147 | /* Convert any pointers to sg list */ |
1148 | unsigned int num_planes; |
1149 | int dir; |
1150 | struct _vk_data *data; |
1151 | |
1152 | /* |
1153 | * check if we are in reset, if so, no buffer transfer is |
1154 | * allowed and return error. |
1155 | */ |
1156 | if (vk->reset_pid) { |
1157 | dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n" , |
1158 | ctx->pid); |
1159 | rc = -EACCES; |
1160 | goto write_free_msgid; |
1161 | } |
1162 | |
1163 | num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK; |
1164 | if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD) |
1165 | dir = DMA_FROM_DEVICE; |
1166 | else |
1167 | dir = DMA_TO_DEVICE; |
1168 | |
1169 | /* Calculate vk_data location */ |
1170 | /* Go to end of the message */ |
1171 | msg_size = entry->to_v_msg[0].size; |
1172 | if (msg_size > entry->to_v_blks) { |
1173 | rc = -EMSGSIZE; |
1174 | goto write_free_msgid; |
1175 | } |
1176 | |
1177 | data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1]; |
1178 | |
1179 | /* Now back up to the start of the pointers */ |
1180 | data -= num_planes; |
1181 | |
1182 | /* Convert user addresses to DMA SG List */ |
1183 | rc = bcm_vk_sg_alloc(dev, dma: entry->dma, dir, vkdata: data, num: num_planes); |
1184 | if (rc) |
1185 | goto write_free_msgid; |
1186 | |
1187 | atomic_inc(v: &ctx->dma_cnt); |
1188 | /* try to embed inband sgl */ |
1189 | sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data, |
1190 | num_planes); |
1191 | entry->to_v_blks += sgl_extra_blks; |
1192 | entry->to_v_msg[0].size += sgl_extra_blks; |
1193 | } else if (entry->to_v_msg[0].function_id == VK_FID_INIT && |
1194 | entry->to_v_msg[0].context_id == VK_NEW_CTX) { |
1195 | /* |
1196 | * Init happens in 2 stages, only the first stage contains the |
1197 | * pid that needs translating. |
1198 | */ |
1199 | pid_t org_pid, pid; |
1200 | |
1201 | /* |
1202 | * translate the pid into the unique host space as user |
1203 | * may run sessions inside containers or process |
1204 | * namespaces. |
1205 | */ |
1206 | #define VK_MSG_PID_MASK 0xffffff00 |
1207 | #define VK_MSG_PID_SH 8 |
1208 | org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK) |
1209 | >> VK_MSG_PID_SH; |
1210 | |
1211 | pid = task_tgid_nr(current); |
1212 | entry->to_v_msg[0].arg = |
1213 | (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) | |
1214 | (pid << VK_MSG_PID_SH); |
1215 | if (org_pid != pid) |
1216 | dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n" , |
1217 | org_pid, org_pid, pid, pid); |
1218 | } |
1219 | |
1220 | /* |
1221 | * store work entry to pending queue until a response is received. |
1222 | * This needs to be done before enqueuing the message |
1223 | */ |
1224 | bcm_vk_append_pendq(chan: &vk->to_v_msg_chan, q_num, entry); |
1225 | |
1226 | rc = bcm_to_v_msg_enqueue(vk, entry); |
1227 | if (rc) { |
1228 | dev_err(dev, "Fail to enqueue msg to to_v queue\n" ); |
1229 | |
1230 | /* remove message from pending list */ |
1231 | entry = bcm_vk_dequeue_pending |
1232 | (vk, |
1233 | chan: &vk->to_v_msg_chan, |
1234 | q_num, |
1235 | msg_id: get_msg_id(msg: &entry->to_v_msg[0])); |
1236 | goto write_free_ent; |
1237 | } |
1238 | |
1239 | return count; |
1240 | |
1241 | write_free_msgid: |
1242 | bcm_vk_msgid_bitmap_clear(vk, start: get_msg_id(msg: &entry->to_v_msg[0]), nbits: 1); |
1243 | write_free_ent: |
1244 | kfree(objp: entry); |
1245 | write_err: |
1246 | return rc; |
1247 | } |
1248 | |
1249 | __poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait) |
1250 | { |
1251 | __poll_t ret = 0; |
1252 | int cnt; |
1253 | struct bcm_vk_ctx *ctx = p_file->private_data; |
1254 | struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); |
1255 | struct device *dev = &vk->pdev->dev; |
1256 | |
1257 | poll_wait(filp: p_file, wait_address: &ctx->rd_wq, p: wait); |
1258 | |
1259 | cnt = atomic_read(v: &ctx->pend_cnt); |
1260 | if (cnt) { |
1261 | ret = (__force __poll_t)(POLLIN | POLLRDNORM); |
1262 | if (cnt < 0) { |
1263 | dev_err(dev, "Error cnt %d, setting back to 0" , cnt); |
1264 | atomic_set(v: &ctx->pend_cnt, i: 0); |
1265 | } |
1266 | } |
1267 | |
1268 | return ret; |
1269 | } |
1270 | |
1271 | int bcm_vk_release(struct inode *inode, struct file *p_file) |
1272 | { |
1273 | int ret; |
1274 | struct bcm_vk_ctx *ctx = p_file->private_data; |
1275 | struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); |
1276 | struct device *dev = &vk->pdev->dev; |
1277 | pid_t pid = ctx->pid; |
1278 | int dma_cnt; |
1279 | unsigned long timeout, start_time; |
1280 | |
1281 | /* |
1282 | * if there are outstanding DMA transactions, need to delay long enough |
1283 | * to ensure that the card side would have stopped touching the host buffer |
1284 | * and its SGL list. A race condition could happen if the host app is killed |
1285 | * abruptly, eg kill -9, while some DMA transfer orders are still inflight. |
1286 | * Nothing could be done except for a delay as host side is running in a |
1287 | * completely async fashion. |
1288 | */ |
1289 | start_time = jiffies; |
1290 | timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS); |
1291 | do { |
1292 | if (time_after(jiffies, timeout)) { |
1293 | dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n" , |
1294 | dma_cnt, ctx->idx, pid); |
1295 | break; |
1296 | } |
1297 | dma_cnt = atomic_read(v: &ctx->dma_cnt); |
1298 | cpu_relax(); |
1299 | cond_resched(); |
1300 | } while (dma_cnt); |
1301 | dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n" , |
1302 | ctx->idx, pid, jiffies_to_msecs(jiffies - start_time)); |
1303 | |
1304 | bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_v_msg_chan, ctx); |
1305 | bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_h_msg_chan, ctx); |
1306 | |
1307 | ret = bcm_vk_free_ctx(vk, ctx); |
1308 | if (ret == 0) |
1309 | ret = bcm_vk_handle_last_sess(vk, pid, q_num: ctx->q_num); |
1310 | else |
1311 | ret = 0; |
1312 | |
1313 | kref_put(kref: &vk->kref, release: bcm_vk_release_data); |
1314 | |
1315 | return ret; |
1316 | } |
1317 | |
1318 | int bcm_vk_msg_init(struct bcm_vk *vk) |
1319 | { |
1320 | struct device *dev = &vk->pdev->dev; |
1321 | int ret; |
1322 | |
1323 | if (bcm_vk_data_init(vk)) { |
1324 | dev_err(dev, "Error initializing internal data structures\n" ); |
1325 | return -EINVAL; |
1326 | } |
1327 | |
1328 | if (bcm_vk_msg_chan_init(chan: &vk->to_v_msg_chan) || |
1329 | bcm_vk_msg_chan_init(chan: &vk->to_h_msg_chan)) { |
1330 | dev_err(dev, "Error initializing communication channel\n" ); |
1331 | return -EIO; |
1332 | } |
1333 | |
1334 | /* read msgq info if ready */ |
1335 | ret = bcm_vk_sync_msgq(vk, force_sync: false); |
1336 | if (ret && (ret != -EAGAIN)) { |
1337 | dev_err(dev, "Error reading comm msg Q info\n" ); |
1338 | return -EIO; |
1339 | } |
1340 | |
1341 | return 0; |
1342 | } |
1343 | |
1344 | void bcm_vk_msg_remove(struct bcm_vk *vk) |
1345 | { |
1346 | bcm_vk_blk_drv_access(vk); |
1347 | |
1348 | /* drain all pending items */ |
1349 | bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_v_msg_chan, NULL); |
1350 | bcm_vk_drain_all_pend(dev: &vk->pdev->dev, chan: &vk->to_h_msg_chan, NULL); |
1351 | } |
1352 | |
1353 | |