1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * Mellanox BlueField SoC TmFifo driver |
4 | * |
5 | * Copyright (C) 2019 Mellanox Technologies |
6 | */ |
7 | |
8 | #include <linux/acpi.h> |
9 | #include <linux/bitfield.h> |
10 | #include <linux/circ_buf.h> |
11 | #include <linux/efi.h> |
12 | #include <linux/irq.h> |
13 | #include <linux/module.h> |
14 | #include <linux/mutex.h> |
15 | #include <linux/platform_device.h> |
16 | #include <linux/types.h> |
17 | |
18 | #include <linux/virtio_config.h> |
19 | #include <linux/virtio_console.h> |
20 | #include <linux/virtio_ids.h> |
21 | #include <linux/virtio_net.h> |
22 | #include <linux/virtio_ring.h> |
23 | |
24 | #include "mlxbf-tmfifo-regs.h" |
25 | |
26 | /* Vring size. */ |
27 | #define MLXBF_TMFIFO_VRING_SIZE SZ_1K |
28 | |
29 | /* Console Tx buffer size. */ |
30 | #define MLXBF_TMFIFO_CON_TX_BUF_SIZE SZ_32K |
31 | |
32 | /* Console Tx buffer reserved space. */ |
33 | #define MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE 8 |
34 | |
35 | /* House-keeping timer interval. */ |
36 | #define MLXBF_TMFIFO_TIMER_INTERVAL (HZ / 10) |
37 | |
38 | /* Virtual devices sharing the TM FIFO. */ |
39 | #define MLXBF_TMFIFO_VDEV_MAX (VIRTIO_ID_CONSOLE + 1) |
40 | |
41 | /* |
42 | * Reserve 1/16 of TmFifo space, so console messages are not starved by |
43 | * the networking traffic. |
44 | */ |
45 | #define MLXBF_TMFIFO_RESERVE_RATIO 16 |
46 | |
47 | /* Message with data needs at least two words (for header & data). */ |
48 | #define MLXBF_TMFIFO_DATA_MIN_WORDS 2 |
49 | |
50 | /* Tx timeout in milliseconds. */ |
51 | #define TMFIFO_TX_TIMEOUT 2000 |
52 | |
53 | /* ACPI UID for BlueField-3. */ |
54 | #define TMFIFO_BF3_UID 1 |
55 | |
56 | struct mlxbf_tmfifo; |
57 | |
58 | /** |
59 | * struct mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring |
60 | * @va: virtual address of the ring |
61 | * @dma: dma address of the ring |
62 | * @vq: pointer to the virtio virtqueue |
63 | * @desc: current descriptor of the pending packet |
64 | * @desc_head: head descriptor of the pending packet |
65 | * @drop_desc: dummy desc for packet dropping |
66 | * @cur_len: processed length of the current descriptor |
67 | * @rem_len: remaining length of the pending packet |
68 | * @rem_padding: remaining bytes to send as paddings |
69 | * @pkt_len: total length of the pending packet |
70 | * @next_avail: next avail descriptor id |
71 | * @num: vring size (number of descriptors) |
72 | * @align: vring alignment size |
73 | * @index: vring index |
74 | * @vdev_id: vring virtio id (VIRTIO_ID_xxx) |
75 | * @tx_timeout: expire time of last tx packet |
76 | * @fifo: pointer to the tmfifo structure |
77 | */ |
78 | struct mlxbf_tmfifo_vring { |
79 | void *va; |
80 | dma_addr_t dma; |
81 | struct virtqueue *vq; |
82 | struct vring_desc *desc; |
83 | struct vring_desc *desc_head; |
84 | struct vring_desc drop_desc; |
85 | int cur_len; |
86 | int rem_len; |
87 | int rem_padding; |
88 | u32 pkt_len; |
89 | u16 next_avail; |
90 | int num; |
91 | int align; |
92 | int index; |
93 | int vdev_id; |
94 | unsigned long tx_timeout; |
95 | struct mlxbf_tmfifo *fifo; |
96 | }; |
97 | |
98 | /* Check whether vring is in drop mode. */ |
99 | #define IS_VRING_DROP(_r) ({ \ |
100 | typeof(_r) (r) = (_r); \ |
101 | r->desc_head == &r->drop_desc; }) |
102 | |
103 | /* A stub length to drop maximum length packet. */ |
104 | #define VRING_DROP_DESC_MAX_LEN GENMASK(15, 0) |
105 | |
106 | /* Interrupt types. */ |
107 | enum { |
108 | MLXBF_TM_RX_LWM_IRQ, |
109 | MLXBF_TM_RX_HWM_IRQ, |
110 | MLXBF_TM_TX_LWM_IRQ, |
111 | MLXBF_TM_TX_HWM_IRQ, |
112 | MLXBF_TM_MAX_IRQ |
113 | }; |
114 | |
115 | /* Ring types (Rx & Tx). */ |
116 | enum { |
117 | MLXBF_TMFIFO_VRING_RX, |
118 | MLXBF_TMFIFO_VRING_TX, |
119 | MLXBF_TMFIFO_VRING_MAX |
120 | }; |
121 | |
122 | /** |
123 | * struct mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device |
124 | * @vdev: virtio device, in which the vdev.id.device field has the |
125 | * VIRTIO_ID_xxx id to distinguish the virtual device. |
126 | * @status: status of the device |
127 | * @features: supported features of the device |
128 | * @vrings: array of tmfifo vrings of this device |
129 | * @config: non-anonymous union for cons and net |
130 | * @config.cons: virtual console config - |
131 | * select if vdev.id.device is VIRTIO_ID_CONSOLE |
132 | * @config.net: virtual network config - |
133 | * select if vdev.id.device is VIRTIO_ID_NET |
134 | * @tx_buf: tx buffer used to buffer data before writing into the FIFO |
135 | */ |
136 | struct mlxbf_tmfifo_vdev { |
137 | struct virtio_device vdev; |
138 | u8 status; |
139 | u64 features; |
140 | struct mlxbf_tmfifo_vring vrings[MLXBF_TMFIFO_VRING_MAX]; |
141 | union { |
142 | struct virtio_console_config cons; |
143 | struct virtio_net_config net; |
144 | } config; |
145 | struct circ_buf tx_buf; |
146 | }; |
147 | |
148 | /** |
149 | * struct mlxbf_tmfifo_irq_info - Structure of the interrupt information |
150 | * @fifo: pointer to the tmfifo structure |
151 | * @irq: interrupt number |
152 | * @index: index into the interrupt array |
153 | */ |
154 | struct mlxbf_tmfifo_irq_info { |
155 | struct mlxbf_tmfifo *fifo; |
156 | int irq; |
157 | int index; |
158 | }; |
159 | |
160 | /** |
161 | * struct mlxbf_tmfifo_io - Structure of the TmFifo IO resource (for both rx & tx) |
162 | * @ctl: control register offset (TMFIFO_RX_CTL / TMFIFO_TX_CTL) |
163 | * @sts: status register offset (TMFIFO_RX_STS / TMFIFO_TX_STS) |
164 | * @data: data register offset (TMFIFO_RX_DATA / TMFIFO_TX_DATA) |
165 | */ |
166 | struct mlxbf_tmfifo_io { |
167 | void __iomem *ctl; |
168 | void __iomem *sts; |
169 | void __iomem *data; |
170 | }; |
171 | |
172 | /** |
173 | * struct mlxbf_tmfifo - Structure of the TmFifo |
174 | * @vdev: array of the virtual devices running over the TmFifo |
175 | * @lock: lock to protect the TmFifo access |
176 | * @res0: mapped resource block 0 |
177 | * @res1: mapped resource block 1 |
178 | * @rx: rx io resource |
179 | * @tx: tx io resource |
180 | * @rx_fifo_size: number of entries of the Rx FIFO |
181 | * @tx_fifo_size: number of entries of the Tx FIFO |
182 | * @pend_events: pending bits for deferred events |
183 | * @irq_info: interrupt information |
184 | * @work: work struct for deferred process |
185 | * @timer: background timer |
186 | * @vring: Tx/Rx ring |
187 | * @spin_lock: Tx/Rx spin lock |
188 | * @is_ready: ready flag |
189 | */ |
190 | struct mlxbf_tmfifo { |
191 | struct mlxbf_tmfifo_vdev *vdev[MLXBF_TMFIFO_VDEV_MAX]; |
192 | struct mutex lock; /* TmFifo lock */ |
193 | void __iomem *res0; |
194 | void __iomem *res1; |
195 | struct mlxbf_tmfifo_io rx; |
196 | struct mlxbf_tmfifo_io tx; |
197 | int rx_fifo_size; |
198 | int tx_fifo_size; |
199 | unsigned long pend_events; |
200 | struct mlxbf_tmfifo_irq_info irq_info[MLXBF_TM_MAX_IRQ]; |
201 | struct work_struct work; |
202 | struct timer_list timer; |
203 | struct mlxbf_tmfifo_vring *vring[2]; |
204 | spinlock_t spin_lock[2]; /* spin lock */ |
205 | bool is_ready; |
206 | }; |
207 | |
208 | /** |
209 | * struct mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header |
210 | * @type: message type |
211 | * @len: payload length in network byte order. Messages sent into the FIFO |
212 | * will be read by the other side as data stream in the same byte order. |
213 | * The length needs to be encoded into network order so both sides |
214 | * could understand it. |
215 | */ |
216 | struct mlxbf_tmfifo_msg_hdr { |
217 | u8 type; |
218 | __be16 len; |
219 | /* private: */ |
220 | u8 unused[5]; |
221 | } __packed __aligned(sizeof(u64)); |
222 | |
223 | /* |
224 | * Default MAC. |
225 | * This MAC address will be read from EFI persistent variable if configured. |
226 | * It can also be reconfigured with standard Linux tools. |
227 | */ |
228 | static u8 mlxbf_tmfifo_net_default_mac[ETH_ALEN] = { |
229 | 0x00, 0x1A, 0xCA, 0xFF, 0xFF, 0x01 |
230 | }; |
231 | |
232 | /* EFI variable name of the MAC address. */ |
233 | static efi_char16_t mlxbf_tmfifo_efi_name[] = L"RshimMacAddr" ; |
234 | |
235 | /* Maximum L2 header length. */ |
236 | #define MLXBF_TMFIFO_NET_L2_OVERHEAD (ETH_HLEN + VLAN_HLEN) |
237 | |
238 | /* Supported virtio-net features. */ |
239 | #define MLXBF_TMFIFO_NET_FEATURES \ |
240 | (BIT_ULL(VIRTIO_NET_F_MTU) | BIT_ULL(VIRTIO_NET_F_STATUS) | \ |
241 | BIT_ULL(VIRTIO_NET_F_MAC)) |
242 | |
243 | #define mlxbf_vdev_to_tmfifo(d) container_of(d, struct mlxbf_tmfifo_vdev, vdev) |
244 | |
245 | /* Free vrings of the FIFO device. */ |
246 | static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo, |
247 | struct mlxbf_tmfifo_vdev *tm_vdev) |
248 | { |
249 | struct mlxbf_tmfifo_vring *vring; |
250 | int i, size; |
251 | |
252 | for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { |
253 | vring = &tm_vdev->vrings[i]; |
254 | if (vring->va) { |
255 | size = vring_size(num: vring->num, align: vring->align); |
256 | dma_free_coherent(dev: tm_vdev->vdev.dev.parent, size, |
257 | cpu_addr: vring->va, dma_handle: vring->dma); |
258 | vring->va = NULL; |
259 | if (vring->vq) { |
260 | vring_del_virtqueue(vq: vring->vq); |
261 | vring->vq = NULL; |
262 | } |
263 | } |
264 | } |
265 | } |
266 | |
267 | /* Allocate vrings for the FIFO. */ |
268 | static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo, |
269 | struct mlxbf_tmfifo_vdev *tm_vdev) |
270 | { |
271 | struct mlxbf_tmfifo_vring *vring; |
272 | struct device *dev; |
273 | dma_addr_t dma; |
274 | int i, size; |
275 | void *va; |
276 | |
277 | for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { |
278 | vring = &tm_vdev->vrings[i]; |
279 | vring->fifo = fifo; |
280 | vring->num = MLXBF_TMFIFO_VRING_SIZE; |
281 | vring->align = SMP_CACHE_BYTES; |
282 | vring->index = i; |
283 | vring->vdev_id = tm_vdev->vdev.id.device; |
284 | vring->drop_desc.len = VRING_DROP_DESC_MAX_LEN; |
285 | dev = &tm_vdev->vdev.dev; |
286 | |
287 | size = vring_size(num: vring->num, align: vring->align); |
288 | va = dma_alloc_coherent(dev: dev->parent, size, dma_handle: &dma, GFP_KERNEL); |
289 | if (!va) { |
290 | mlxbf_tmfifo_free_vrings(fifo, tm_vdev); |
291 | dev_err(dev->parent, "dma_alloc_coherent failed\n" ); |
292 | return -ENOMEM; |
293 | } |
294 | |
295 | vring->va = va; |
296 | vring->dma = dma; |
297 | } |
298 | |
299 | return 0; |
300 | } |
301 | |
302 | /* Disable interrupts of the FIFO device. */ |
303 | static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo) |
304 | { |
305 | int i, irq; |
306 | |
307 | for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) { |
308 | irq = fifo->irq_info[i].irq; |
309 | fifo->irq_info[i].irq = 0; |
310 | disable_irq(irq); |
311 | } |
312 | } |
313 | |
314 | /* Interrupt handler. */ |
315 | static irqreturn_t mlxbf_tmfifo_irq_handler(int irq, void *arg) |
316 | { |
317 | struct mlxbf_tmfifo_irq_info *irq_info = arg; |
318 | |
319 | if (!test_and_set_bit(nr: irq_info->index, addr: &irq_info->fifo->pend_events)) |
320 | schedule_work(work: &irq_info->fifo->work); |
321 | |
322 | return IRQ_HANDLED; |
323 | } |
324 | |
325 | /* Get the next packet descriptor from the vring. */ |
326 | static struct vring_desc * |
327 | mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring) |
328 | { |
329 | const struct vring *vr = virtqueue_get_vring(vq: vring->vq); |
330 | struct virtio_device *vdev = vring->vq->vdev; |
331 | unsigned int idx, head; |
332 | |
333 | if (vring->next_avail == virtio16_to_cpu(vdev, val: vr->avail->idx)) |
334 | return NULL; |
335 | |
336 | /* Make sure 'avail->idx' is visible already. */ |
337 | virtio_rmb(weak_barriers: false); |
338 | |
339 | idx = vring->next_avail % vr->num; |
340 | head = virtio16_to_cpu(vdev, val: vr->avail->ring[idx]); |
341 | if (WARN_ON(head >= vr->num)) |
342 | return NULL; |
343 | |
344 | vring->next_avail++; |
345 | |
346 | return &vr->desc[head]; |
347 | } |
348 | |
349 | /* Release virtio descriptor. */ |
350 | static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring, |
351 | struct vring_desc *desc, u32 len) |
352 | { |
353 | const struct vring *vr = virtqueue_get_vring(vq: vring->vq); |
354 | struct virtio_device *vdev = vring->vq->vdev; |
355 | u16 idx, vr_idx; |
356 | |
357 | vr_idx = virtio16_to_cpu(vdev, val: vr->used->idx); |
358 | idx = vr_idx % vr->num; |
359 | vr->used->ring[idx].id = cpu_to_virtio32(vdev, val: desc - vr->desc); |
360 | vr->used->ring[idx].len = cpu_to_virtio32(vdev, val: len); |
361 | |
362 | /* |
363 | * Virtio could poll and check the 'idx' to decide whether the desc is |
364 | * done or not. Add a memory barrier here to make sure the update above |
365 | * completes before updating the idx. |
366 | */ |
367 | virtio_mb(weak_barriers: false); |
368 | vr->used->idx = cpu_to_virtio16(vdev, val: vr_idx + 1); |
369 | } |
370 | |
371 | /* Get the total length of the descriptor chain. */ |
372 | static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring, |
373 | struct vring_desc *desc) |
374 | { |
375 | const struct vring *vr = virtqueue_get_vring(vq: vring->vq); |
376 | struct virtio_device *vdev = vring->vq->vdev; |
377 | u32 len = 0, idx; |
378 | |
379 | while (desc) { |
380 | len += virtio32_to_cpu(vdev, val: desc->len); |
381 | if (!(virtio16_to_cpu(vdev, val: desc->flags) & VRING_DESC_F_NEXT)) |
382 | break; |
383 | idx = virtio16_to_cpu(vdev, val: desc->next); |
384 | desc = &vr->desc[idx]; |
385 | } |
386 | |
387 | return len; |
388 | } |
389 | |
390 | static void mlxbf_tmfifo_release_pkt(struct mlxbf_tmfifo_vring *vring) |
391 | { |
392 | struct vring_desc *desc_head; |
393 | u32 len = 0; |
394 | |
395 | if (vring->desc_head) { |
396 | desc_head = vring->desc_head; |
397 | len = vring->pkt_len; |
398 | } else { |
399 | desc_head = mlxbf_tmfifo_get_next_desc(vring); |
400 | len = mlxbf_tmfifo_get_pkt_len(vring, desc: desc_head); |
401 | } |
402 | |
403 | if (desc_head) |
404 | mlxbf_tmfifo_release_desc(vring, desc: desc_head, len); |
405 | |
406 | vring->pkt_len = 0; |
407 | vring->desc = NULL; |
408 | vring->desc_head = NULL; |
409 | } |
410 | |
411 | static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring, |
412 | struct vring_desc *desc, bool is_rx) |
413 | { |
414 | struct virtio_device *vdev = vring->vq->vdev; |
415 | struct virtio_net_hdr *net_hdr; |
416 | |
417 | net_hdr = phys_to_virt(address: virtio64_to_cpu(vdev, val: desc->addr)); |
418 | memset(net_hdr, 0, sizeof(*net_hdr)); |
419 | } |
420 | |
421 | /* Get and initialize the next packet. */ |
422 | static struct vring_desc * |
423 | mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx) |
424 | { |
425 | struct vring_desc *desc; |
426 | |
427 | desc = mlxbf_tmfifo_get_next_desc(vring); |
428 | if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET) |
429 | mlxbf_tmfifo_init_net_desc(vring, desc, is_rx); |
430 | |
431 | vring->desc_head = desc; |
432 | vring->desc = desc; |
433 | |
434 | return desc; |
435 | } |
436 | |
437 | /* House-keeping timer. */ |
438 | static void mlxbf_tmfifo_timer(struct timer_list *t) |
439 | { |
440 | struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer); |
441 | int rx, tx; |
442 | |
443 | rx = !test_and_set_bit(nr: MLXBF_TM_RX_HWM_IRQ, addr: &fifo->pend_events); |
444 | tx = !test_and_set_bit(nr: MLXBF_TM_TX_LWM_IRQ, addr: &fifo->pend_events); |
445 | |
446 | if (rx || tx) |
447 | schedule_work(work: &fifo->work); |
448 | |
449 | mod_timer(timer: &fifo->timer, expires: jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); |
450 | } |
451 | |
452 | /* Copy one console packet into the output buffer. */ |
453 | static void mlxbf_tmfifo_console_output_one(struct mlxbf_tmfifo_vdev *cons, |
454 | struct mlxbf_tmfifo_vring *vring, |
455 | struct vring_desc *desc) |
456 | { |
457 | const struct vring *vr = virtqueue_get_vring(vq: vring->vq); |
458 | struct virtio_device *vdev = &cons->vdev; |
459 | u32 len, idx, seg; |
460 | void *addr; |
461 | |
462 | while (desc) { |
463 | addr = phys_to_virt(address: virtio64_to_cpu(vdev, val: desc->addr)); |
464 | len = virtio32_to_cpu(vdev, val: desc->len); |
465 | |
466 | seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail, |
467 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); |
468 | if (len <= seg) { |
469 | memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len); |
470 | } else { |
471 | memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg); |
472 | addr += seg; |
473 | memcpy(cons->tx_buf.buf, addr, len - seg); |
474 | } |
475 | cons->tx_buf.head = (cons->tx_buf.head + len) % |
476 | MLXBF_TMFIFO_CON_TX_BUF_SIZE; |
477 | |
478 | if (!(virtio16_to_cpu(vdev, val: desc->flags) & VRING_DESC_F_NEXT)) |
479 | break; |
480 | idx = virtio16_to_cpu(vdev, val: desc->next); |
481 | desc = &vr->desc[idx]; |
482 | } |
483 | } |
484 | |
485 | /* Copy console data into the output buffer. */ |
486 | static void mlxbf_tmfifo_console_output(struct mlxbf_tmfifo_vdev *cons, |
487 | struct mlxbf_tmfifo_vring *vring) |
488 | { |
489 | struct vring_desc *desc; |
490 | u32 len, avail; |
491 | |
492 | desc = mlxbf_tmfifo_get_next_desc(vring); |
493 | while (desc) { |
494 | /* Release the packet if not enough space. */ |
495 | len = mlxbf_tmfifo_get_pkt_len(vring, desc); |
496 | avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail, |
497 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); |
498 | if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) { |
499 | mlxbf_tmfifo_release_desc(vring, desc, len); |
500 | break; |
501 | } |
502 | |
503 | mlxbf_tmfifo_console_output_one(cons, vring, desc); |
504 | mlxbf_tmfifo_release_desc(vring, desc, len); |
505 | desc = mlxbf_tmfifo_get_next_desc(vring); |
506 | } |
507 | } |
508 | |
509 | /* Get the number of available words in Rx FIFO for receiving. */ |
510 | static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo) |
511 | { |
512 | u64 sts; |
513 | |
514 | sts = readq(addr: fifo->rx.sts); |
515 | return FIELD_GET(MLXBF_TMFIFO_RX_STS__COUNT_MASK, sts); |
516 | } |
517 | |
518 | /* Get the number of available words in the TmFifo for sending. */ |
519 | static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id) |
520 | { |
521 | int tx_reserve; |
522 | u32 count; |
523 | u64 sts; |
524 | |
525 | /* Reserve some room in FIFO for console messages. */ |
526 | if (vdev_id == VIRTIO_ID_NET) |
527 | tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO; |
528 | else |
529 | tx_reserve = 1; |
530 | |
531 | sts = readq(addr: fifo->tx.sts); |
532 | count = FIELD_GET(MLXBF_TMFIFO_TX_STS__COUNT_MASK, sts); |
533 | return fifo->tx_fifo_size - tx_reserve - count; |
534 | } |
535 | |
536 | /* Console Tx (move data from the output buffer into the TmFifo). */ |
537 | static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail) |
538 | { |
539 | struct mlxbf_tmfifo_msg_hdr hdr; |
540 | struct mlxbf_tmfifo_vdev *cons; |
541 | unsigned long flags; |
542 | int size, seg; |
543 | void *addr; |
544 | u64 data; |
545 | |
546 | /* Return if not enough space available. */ |
547 | if (avail < MLXBF_TMFIFO_DATA_MIN_WORDS) |
548 | return; |
549 | |
550 | cons = fifo->vdev[VIRTIO_ID_CONSOLE]; |
551 | if (!cons || !cons->tx_buf.buf) |
552 | return; |
553 | |
554 | /* Return if no data to send. */ |
555 | size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail, |
556 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); |
557 | if (size == 0) |
558 | return; |
559 | |
560 | /* Adjust the size to available space. */ |
561 | if (size + sizeof(hdr) > avail * sizeof(u64)) |
562 | size = avail * sizeof(u64) - sizeof(hdr); |
563 | |
564 | /* Write header. */ |
565 | hdr.type = VIRTIO_ID_CONSOLE; |
566 | hdr.len = htons(size); |
567 | writeq(val: *(u64 *)&hdr, addr: fifo->tx.data); |
568 | |
569 | /* Use spin-lock to protect the 'cons->tx_buf'. */ |
570 | spin_lock_irqsave(&fifo->spin_lock[0], flags); |
571 | |
572 | while (size > 0) { |
573 | addr = cons->tx_buf.buf + cons->tx_buf.tail; |
574 | |
575 | seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail, |
576 | MLXBF_TMFIFO_CON_TX_BUF_SIZE); |
577 | if (seg >= sizeof(u64)) { |
578 | memcpy(&data, addr, sizeof(u64)); |
579 | } else { |
580 | memcpy(&data, addr, seg); |
581 | memcpy((u8 *)&data + seg, cons->tx_buf.buf, |
582 | sizeof(u64) - seg); |
583 | } |
584 | writeq(val: data, addr: fifo->tx.data); |
585 | |
586 | if (size >= sizeof(u64)) { |
587 | cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) % |
588 | MLXBF_TMFIFO_CON_TX_BUF_SIZE; |
589 | size -= sizeof(u64); |
590 | } else { |
591 | cons->tx_buf.tail = (cons->tx_buf.tail + size) % |
592 | MLXBF_TMFIFO_CON_TX_BUF_SIZE; |
593 | size = 0; |
594 | } |
595 | } |
596 | |
597 | spin_unlock_irqrestore(lock: &fifo->spin_lock[0], flags); |
598 | } |
599 | |
600 | /* Rx/Tx one word in the descriptor buffer. */ |
601 | static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring, |
602 | struct vring_desc *desc, |
603 | bool is_rx, int len) |
604 | { |
605 | struct virtio_device *vdev = vring->vq->vdev; |
606 | struct mlxbf_tmfifo *fifo = vring->fifo; |
607 | void *addr; |
608 | u64 data; |
609 | |
610 | /* Get the buffer address of this desc. */ |
611 | addr = phys_to_virt(address: virtio64_to_cpu(vdev, val: desc->addr)); |
612 | |
613 | /* Read a word from FIFO for Rx. */ |
614 | if (is_rx) |
615 | data = readq(addr: fifo->rx.data); |
616 | |
617 | if (vring->cur_len + sizeof(u64) <= len) { |
618 | /* The whole word. */ |
619 | if (is_rx) { |
620 | if (!IS_VRING_DROP(vring)) |
621 | memcpy(addr + vring->cur_len, &data, |
622 | sizeof(u64)); |
623 | } else { |
624 | memcpy(&data, addr + vring->cur_len, |
625 | sizeof(u64)); |
626 | } |
627 | vring->cur_len += sizeof(u64); |
628 | } else { |
629 | /* Leftover bytes. */ |
630 | if (is_rx) { |
631 | if (!IS_VRING_DROP(vring)) |
632 | memcpy(addr + vring->cur_len, &data, |
633 | len - vring->cur_len); |
634 | } else { |
635 | data = 0; |
636 | memcpy(&data, addr + vring->cur_len, |
637 | len - vring->cur_len); |
638 | } |
639 | vring->cur_len = len; |
640 | } |
641 | |
642 | /* Write the word into FIFO for Tx. */ |
643 | if (!is_rx) |
644 | writeq(val: data, addr: fifo->tx.data); |
645 | } |
646 | |
647 | /* |
648 | * Rx/Tx packet header. |
649 | * |
650 | * In Rx case, the packet might be found to belong to a different vring since |
651 | * the TmFifo is shared by different services. In such case, the 'vring_change' |
652 | * flag is set. |
653 | */ |
654 | static void (struct mlxbf_tmfifo_vring *vring, |
655 | struct vring_desc **desc, |
656 | bool is_rx, bool *vring_change) |
657 | { |
658 | struct mlxbf_tmfifo *fifo = vring->fifo; |
659 | struct virtio_net_config *config; |
660 | struct mlxbf_tmfifo_msg_hdr hdr; |
661 | int vdev_id, hdr_len; |
662 | bool drop_rx = false; |
663 | |
664 | /* Read/Write packet header. */ |
665 | if (is_rx) { |
666 | /* Drain one word from the FIFO. */ |
667 | *(u64 *)&hdr = readq(addr: fifo->rx.data); |
668 | |
669 | /* Skip the length 0 packets (keepalive). */ |
670 | if (hdr.len == 0) |
671 | return; |
672 | |
673 | /* Check packet type. */ |
674 | if (hdr.type == VIRTIO_ID_NET) { |
675 | vdev_id = VIRTIO_ID_NET; |
676 | hdr_len = sizeof(struct virtio_net_hdr); |
677 | config = &fifo->vdev[vdev_id]->config.net; |
678 | /* A legacy-only interface for now. */ |
679 | if (ntohs(hdr.len) > |
680 | __virtio16_to_cpu(little_endian: virtio_legacy_is_little_endian(), |
681 | val: config->mtu) + |
682 | MLXBF_TMFIFO_NET_L2_OVERHEAD) |
683 | drop_rx = true; |
684 | } else { |
685 | vdev_id = VIRTIO_ID_CONSOLE; |
686 | hdr_len = 0; |
687 | } |
688 | |
689 | /* |
690 | * Check whether the new packet still belongs to this vring. |
691 | * If not, update the pkt_len of the new vring. |
692 | */ |
693 | if (vdev_id != vring->vdev_id) { |
694 | struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id]; |
695 | |
696 | if (!tm_dev2) |
697 | return; |
698 | vring->desc = *desc; |
699 | vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX]; |
700 | *vring_change = true; |
701 | } |
702 | |
703 | if (drop_rx && !IS_VRING_DROP(vring)) { |
704 | if (vring->desc_head) |
705 | mlxbf_tmfifo_release_pkt(vring); |
706 | *desc = &vring->drop_desc; |
707 | vring->desc_head = *desc; |
708 | vring->desc = *desc; |
709 | } |
710 | |
711 | vring->pkt_len = ntohs(hdr.len) + hdr_len; |
712 | } else { |
713 | /* Network virtio has an extra header. */ |
714 | hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ? |
715 | sizeof(struct virtio_net_hdr) : 0; |
716 | vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc: *desc); |
717 | hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ? |
718 | VIRTIO_ID_NET : VIRTIO_ID_CONSOLE; |
719 | hdr.len = htons(vring->pkt_len - hdr_len); |
720 | writeq(val: *(u64 *)&hdr, addr: fifo->tx.data); |
721 | } |
722 | |
723 | vring->cur_len = hdr_len; |
724 | vring->rem_len = vring->pkt_len; |
725 | fifo->vring[is_rx] = vring; |
726 | } |
727 | |
728 | /* |
729 | * Rx/Tx one descriptor. |
730 | * |
731 | * Return true to indicate more data available. |
732 | */ |
733 | static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring, |
734 | bool is_rx, int *avail) |
735 | { |
736 | const struct vring *vr = virtqueue_get_vring(vq: vring->vq); |
737 | struct mlxbf_tmfifo *fifo = vring->fifo; |
738 | struct virtio_device *vdev; |
739 | bool vring_change = false; |
740 | struct vring_desc *desc; |
741 | unsigned long flags; |
742 | u32 len, idx; |
743 | |
744 | vdev = &fifo->vdev[vring->vdev_id]->vdev; |
745 | |
746 | /* Get the descriptor of the next packet. */ |
747 | if (!vring->desc) { |
748 | desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx); |
749 | if (!desc) { |
750 | /* Drop next Rx packet to avoid stuck. */ |
751 | if (is_rx) { |
752 | desc = &vring->drop_desc; |
753 | vring->desc_head = desc; |
754 | vring->desc = desc; |
755 | } else { |
756 | return false; |
757 | } |
758 | } |
759 | } else { |
760 | desc = vring->desc; |
761 | } |
762 | |
763 | /* Beginning of a packet. Start to Rx/Tx packet header. */ |
764 | if (vring->pkt_len == 0) { |
765 | mlxbf_tmfifo_rxtx_header(vring, desc: &desc, is_rx, vring_change: &vring_change); |
766 | (*avail)--; |
767 | |
768 | /* Return if new packet is for another ring. */ |
769 | if (vring_change) |
770 | return false; |
771 | goto mlxbf_tmfifo_desc_done; |
772 | } |
773 | |
774 | /* Get the length of this desc. */ |
775 | len = virtio32_to_cpu(vdev, val: desc->len); |
776 | if (len > vring->rem_len) |
777 | len = vring->rem_len; |
778 | |
779 | /* Rx/Tx one word (8 bytes) if not done. */ |
780 | if (vring->cur_len < len) { |
781 | mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len); |
782 | (*avail)--; |
783 | } |
784 | |
785 | /* Check again whether it's done. */ |
786 | if (vring->cur_len == len) { |
787 | vring->cur_len = 0; |
788 | vring->rem_len -= len; |
789 | |
790 | /* Get the next desc on the chain. */ |
791 | if (!IS_VRING_DROP(vring) && vring->rem_len > 0 && |
792 | (virtio16_to_cpu(vdev, val: desc->flags) & VRING_DESC_F_NEXT)) { |
793 | idx = virtio16_to_cpu(vdev, val: desc->next); |
794 | desc = &vr->desc[idx]; |
795 | goto mlxbf_tmfifo_desc_done; |
796 | } |
797 | |
798 | /* Done and release the packet. */ |
799 | desc = NULL; |
800 | fifo->vring[is_rx] = NULL; |
801 | if (!IS_VRING_DROP(vring)) { |
802 | mlxbf_tmfifo_release_pkt(vring); |
803 | } else { |
804 | vring->pkt_len = 0; |
805 | vring->desc_head = NULL; |
806 | vring->desc = NULL; |
807 | return false; |
808 | } |
809 | |
810 | /* |
811 | * Make sure the load/store are in order before |
812 | * returning back to virtio. |
813 | */ |
814 | virtio_mb(weak_barriers: false); |
815 | |
816 | /* Notify upper layer that packet is done. */ |
817 | spin_lock_irqsave(&fifo->spin_lock[is_rx], flags); |
818 | vring_interrupt(irq: 0, vq: vring->vq); |
819 | spin_unlock_irqrestore(lock: &fifo->spin_lock[is_rx], flags); |
820 | } |
821 | |
822 | mlxbf_tmfifo_desc_done: |
823 | /* Save the current desc. */ |
824 | vring->desc = desc; |
825 | |
826 | return true; |
827 | } |
828 | |
829 | static void mlxbf_tmfifo_check_tx_timeout(struct mlxbf_tmfifo_vring *vring) |
830 | { |
831 | unsigned long flags; |
832 | |
833 | /* Only handle Tx timeout for network vdev. */ |
834 | if (vring->vdev_id != VIRTIO_ID_NET) |
835 | return; |
836 | |
837 | /* Initialize the timeout or return if not expired. */ |
838 | if (!vring->tx_timeout) { |
839 | /* Initialize the timeout. */ |
840 | vring->tx_timeout = jiffies + |
841 | msecs_to_jiffies(TMFIFO_TX_TIMEOUT); |
842 | return; |
843 | } else if (time_before(jiffies, vring->tx_timeout)) { |
844 | /* Return if not timeout yet. */ |
845 | return; |
846 | } |
847 | |
848 | /* |
849 | * Drop the packet after timeout. The outstanding packet is |
850 | * released and the remaining bytes will be sent with padding byte 0x00 |
851 | * as a recovery. On the peer(host) side, the padding bytes 0x00 will be |
852 | * either dropped directly, or appended into existing outstanding packet |
853 | * thus dropped as corrupted network packet. |
854 | */ |
855 | vring->rem_padding = round_up(vring->rem_len, sizeof(u64)); |
856 | mlxbf_tmfifo_release_pkt(vring); |
857 | vring->cur_len = 0; |
858 | vring->rem_len = 0; |
859 | vring->fifo->vring[0] = NULL; |
860 | |
861 | /* |
862 | * Make sure the load/store are in order before |
863 | * returning back to virtio. |
864 | */ |
865 | virtio_mb(weak_barriers: false); |
866 | |
867 | /* Notify upper layer. */ |
868 | spin_lock_irqsave(&vring->fifo->spin_lock[0], flags); |
869 | vring_interrupt(irq: 0, vq: vring->vq); |
870 | spin_unlock_irqrestore(lock: &vring->fifo->spin_lock[0], flags); |
871 | } |
872 | |
873 | /* Rx & Tx processing of a queue. */ |
874 | static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx) |
875 | { |
876 | int avail = 0, devid = vring->vdev_id; |
877 | struct mlxbf_tmfifo *fifo; |
878 | bool more; |
879 | |
880 | fifo = vring->fifo; |
881 | |
882 | /* Return if vdev is not ready. */ |
883 | if (!fifo || !fifo->vdev[devid]) |
884 | return; |
885 | |
886 | /* Return if another vring is running. */ |
887 | if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring) |
888 | return; |
889 | |
890 | /* Only handle console and network for now. */ |
891 | if (WARN_ON(devid != VIRTIO_ID_NET && devid != VIRTIO_ID_CONSOLE)) |
892 | return; |
893 | |
894 | do { |
895 | retry: |
896 | /* Get available FIFO space. */ |
897 | if (avail == 0) { |
898 | if (is_rx) |
899 | avail = mlxbf_tmfifo_get_rx_avail(fifo); |
900 | else |
901 | avail = mlxbf_tmfifo_get_tx_avail(fifo, vdev_id: devid); |
902 | if (avail <= 0) |
903 | break; |
904 | } |
905 | |
906 | /* Insert paddings for discarded Tx packet. */ |
907 | if (!is_rx) { |
908 | vring->tx_timeout = 0; |
909 | while (vring->rem_padding >= sizeof(u64)) { |
910 | writeq(val: 0, addr: vring->fifo->tx.data); |
911 | vring->rem_padding -= sizeof(u64); |
912 | if (--avail == 0) |
913 | goto retry; |
914 | } |
915 | } |
916 | |
917 | /* Console output always comes from the Tx buffer. */ |
918 | if (!is_rx && devid == VIRTIO_ID_CONSOLE) { |
919 | mlxbf_tmfifo_console_tx(fifo, avail); |
920 | break; |
921 | } |
922 | |
923 | /* Handle one descriptor. */ |
924 | more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, avail: &avail); |
925 | } while (more); |
926 | |
927 | /* Check Tx timeout. */ |
928 | if (avail <= 0 && !is_rx) |
929 | mlxbf_tmfifo_check_tx_timeout(vring); |
930 | } |
931 | |
932 | /* Handle Rx or Tx queues. */ |
933 | static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id, |
934 | int irq_id, bool is_rx) |
935 | { |
936 | struct mlxbf_tmfifo_vdev *tm_vdev; |
937 | struct mlxbf_tmfifo_vring *vring; |
938 | int i; |
939 | |
940 | if (!test_and_clear_bit(nr: irq_id, addr: &fifo->pend_events) || |
941 | !fifo->irq_info[irq_id].irq) |
942 | return; |
943 | |
944 | for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) { |
945 | tm_vdev = fifo->vdev[i]; |
946 | if (tm_vdev) { |
947 | vring = &tm_vdev->vrings[queue_id]; |
948 | if (vring->vq) |
949 | mlxbf_tmfifo_rxtx(vring, is_rx); |
950 | } |
951 | } |
952 | } |
953 | |
954 | /* Work handler for Rx and Tx case. */ |
955 | static void mlxbf_tmfifo_work_handler(struct work_struct *work) |
956 | { |
957 | struct mlxbf_tmfifo *fifo; |
958 | |
959 | fifo = container_of(work, struct mlxbf_tmfifo, work); |
960 | if (!fifo->is_ready) |
961 | return; |
962 | |
963 | mutex_lock(&fifo->lock); |
964 | |
965 | /* Tx (Send data to the TmFifo). */ |
966 | mlxbf_tmfifo_work_rxtx(fifo, queue_id: MLXBF_TMFIFO_VRING_TX, |
967 | irq_id: MLXBF_TM_TX_LWM_IRQ, is_rx: false); |
968 | |
969 | /* Rx (Receive data from the TmFifo). */ |
970 | mlxbf_tmfifo_work_rxtx(fifo, queue_id: MLXBF_TMFIFO_VRING_RX, |
971 | irq_id: MLXBF_TM_RX_HWM_IRQ, is_rx: true); |
972 | |
973 | mutex_unlock(lock: &fifo->lock); |
974 | } |
975 | |
976 | /* The notify function is called when new buffers are posted. */ |
977 | static bool mlxbf_tmfifo_virtio_notify(struct virtqueue *vq) |
978 | { |
979 | struct mlxbf_tmfifo_vring *vring = vq->priv; |
980 | struct mlxbf_tmfifo_vdev *tm_vdev; |
981 | struct mlxbf_tmfifo *fifo; |
982 | unsigned long flags; |
983 | |
984 | fifo = vring->fifo; |
985 | |
986 | /* |
987 | * Virtio maintains vrings in pairs, even number ring for Rx |
988 | * and odd number ring for Tx. |
989 | */ |
990 | if (vring->index & BIT(0)) { |
991 | /* |
992 | * Console could make blocking call with interrupts disabled. |
993 | * In such case, the vring needs to be served right away. For |
994 | * other cases, just set the TX LWM bit to start Tx in the |
995 | * worker handler. |
996 | */ |
997 | if (vring->vdev_id == VIRTIO_ID_CONSOLE) { |
998 | spin_lock_irqsave(&fifo->spin_lock[0], flags); |
999 | tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE]; |
1000 | mlxbf_tmfifo_console_output(cons: tm_vdev, vring); |
1001 | spin_unlock_irqrestore(lock: &fifo->spin_lock[0], flags); |
1002 | set_bit(nr: MLXBF_TM_TX_LWM_IRQ, addr: &fifo->pend_events); |
1003 | } else if (test_and_set_bit(nr: MLXBF_TM_TX_LWM_IRQ, |
1004 | addr: &fifo->pend_events)) { |
1005 | return true; |
1006 | } |
1007 | } else { |
1008 | if (test_and_set_bit(nr: MLXBF_TM_RX_HWM_IRQ, addr: &fifo->pend_events)) |
1009 | return true; |
1010 | } |
1011 | |
1012 | schedule_work(work: &fifo->work); |
1013 | |
1014 | return true; |
1015 | } |
1016 | |
1017 | /* Get the array of feature bits for this device. */ |
1018 | static u64 mlxbf_tmfifo_virtio_get_features(struct virtio_device *vdev) |
1019 | { |
1020 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1021 | |
1022 | return tm_vdev->features; |
1023 | } |
1024 | |
1025 | /* Confirm device features to use. */ |
1026 | static int mlxbf_tmfifo_virtio_finalize_features(struct virtio_device *vdev) |
1027 | { |
1028 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1029 | |
1030 | tm_vdev->features = vdev->features; |
1031 | |
1032 | return 0; |
1033 | } |
1034 | |
1035 | /* Free virtqueues found by find_vqs(). */ |
1036 | static void mlxbf_tmfifo_virtio_del_vqs(struct virtio_device *vdev) |
1037 | { |
1038 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1039 | struct mlxbf_tmfifo_vring *vring; |
1040 | struct virtqueue *vq; |
1041 | int i; |
1042 | |
1043 | for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { |
1044 | vring = &tm_vdev->vrings[i]; |
1045 | |
1046 | /* Release the pending packet. */ |
1047 | if (vring->desc) |
1048 | mlxbf_tmfifo_release_pkt(vring); |
1049 | vq = vring->vq; |
1050 | if (vq) { |
1051 | vring->vq = NULL; |
1052 | vring_del_virtqueue(vq); |
1053 | } |
1054 | } |
1055 | } |
1056 | |
1057 | /* Create and initialize the virtual queues. */ |
1058 | static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, |
1059 | unsigned int nvqs, |
1060 | struct virtqueue *vqs[], |
1061 | vq_callback_t *callbacks[], |
1062 | const char * const names[], |
1063 | const bool *ctx, |
1064 | struct irq_affinity *desc) |
1065 | { |
1066 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1067 | struct mlxbf_tmfifo_vring *vring; |
1068 | struct virtqueue *vq; |
1069 | int i, ret, size; |
1070 | |
1071 | if (nvqs > ARRAY_SIZE(tm_vdev->vrings)) |
1072 | return -EINVAL; |
1073 | |
1074 | for (i = 0; i < nvqs; ++i) { |
1075 | if (!names[i]) { |
1076 | ret = -EINVAL; |
1077 | goto error; |
1078 | } |
1079 | vring = &tm_vdev->vrings[i]; |
1080 | |
1081 | /* zero vring */ |
1082 | size = vring_size(num: vring->num, align: vring->align); |
1083 | memset(vring->va, 0, size); |
1084 | vq = vring_new_virtqueue(index: i, num: vring->num, vring_align: vring->align, vdev, |
1085 | weak_barriers: false, ctx: false, pages: vring->va, |
1086 | notify: mlxbf_tmfifo_virtio_notify, |
1087 | callback: callbacks[i], name: names[i]); |
1088 | if (!vq) { |
1089 | dev_err(&vdev->dev, "vring_new_virtqueue failed\n" ); |
1090 | ret = -ENOMEM; |
1091 | goto error; |
1092 | } |
1093 | |
1094 | vq->num_max = vring->num; |
1095 | |
1096 | vq->priv = vring; |
1097 | |
1098 | /* Make vq update visible before using it. */ |
1099 | virtio_mb(weak_barriers: false); |
1100 | |
1101 | vqs[i] = vq; |
1102 | vring->vq = vq; |
1103 | } |
1104 | |
1105 | return 0; |
1106 | |
1107 | error: |
1108 | mlxbf_tmfifo_virtio_del_vqs(vdev); |
1109 | return ret; |
1110 | } |
1111 | |
1112 | /* Read the status byte. */ |
1113 | static u8 mlxbf_tmfifo_virtio_get_status(struct virtio_device *vdev) |
1114 | { |
1115 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1116 | |
1117 | return tm_vdev->status; |
1118 | } |
1119 | |
1120 | /* Write the status byte. */ |
1121 | static void mlxbf_tmfifo_virtio_set_status(struct virtio_device *vdev, |
1122 | u8 status) |
1123 | { |
1124 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1125 | |
1126 | tm_vdev->status = status; |
1127 | } |
1128 | |
1129 | /* Reset the device. Not much here for now. */ |
1130 | static void mlxbf_tmfifo_virtio_reset(struct virtio_device *vdev) |
1131 | { |
1132 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1133 | |
1134 | tm_vdev->status = 0; |
1135 | } |
1136 | |
1137 | /* Read the value of a configuration field. */ |
1138 | static void mlxbf_tmfifo_virtio_get(struct virtio_device *vdev, |
1139 | unsigned int offset, |
1140 | void *buf, |
1141 | unsigned int len) |
1142 | { |
1143 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1144 | |
1145 | if ((u64)offset + len > sizeof(tm_vdev->config)) |
1146 | return; |
1147 | |
1148 | memcpy(buf, (u8 *)&tm_vdev->config + offset, len); |
1149 | } |
1150 | |
1151 | /* Write the value of a configuration field. */ |
1152 | static void mlxbf_tmfifo_virtio_set(struct virtio_device *vdev, |
1153 | unsigned int offset, |
1154 | const void *buf, |
1155 | unsigned int len) |
1156 | { |
1157 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1158 | |
1159 | if ((u64)offset + len > sizeof(tm_vdev->config)) |
1160 | return; |
1161 | |
1162 | memcpy((u8 *)&tm_vdev->config + offset, buf, len); |
1163 | } |
1164 | |
1165 | static void tmfifo_virtio_dev_release(struct device *device) |
1166 | { |
1167 | struct virtio_device *vdev = |
1168 | container_of(device, struct virtio_device, dev); |
1169 | struct mlxbf_tmfifo_vdev *tm_vdev = mlxbf_vdev_to_tmfifo(vdev); |
1170 | |
1171 | kfree(objp: tm_vdev); |
1172 | } |
1173 | |
1174 | /* Virtio config operations. */ |
1175 | static const struct virtio_config_ops mlxbf_tmfifo_virtio_config_ops = { |
1176 | .get_features = mlxbf_tmfifo_virtio_get_features, |
1177 | .finalize_features = mlxbf_tmfifo_virtio_finalize_features, |
1178 | .find_vqs = mlxbf_tmfifo_virtio_find_vqs, |
1179 | .del_vqs = mlxbf_tmfifo_virtio_del_vqs, |
1180 | .reset = mlxbf_tmfifo_virtio_reset, |
1181 | .set_status = mlxbf_tmfifo_virtio_set_status, |
1182 | .get_status = mlxbf_tmfifo_virtio_get_status, |
1183 | .get = mlxbf_tmfifo_virtio_get, |
1184 | .set = mlxbf_tmfifo_virtio_set, |
1185 | }; |
1186 | |
1187 | /* Create vdev for the FIFO. */ |
1188 | static int mlxbf_tmfifo_create_vdev(struct device *dev, |
1189 | struct mlxbf_tmfifo *fifo, |
1190 | int vdev_id, u64 features, |
1191 | void *config, u32 size) |
1192 | { |
1193 | struct mlxbf_tmfifo_vdev *tm_vdev, *reg_dev = NULL; |
1194 | int ret; |
1195 | |
1196 | mutex_lock(&fifo->lock); |
1197 | |
1198 | tm_vdev = fifo->vdev[vdev_id]; |
1199 | if (tm_vdev) { |
1200 | dev_err(dev, "vdev %d already exists\n" , vdev_id); |
1201 | ret = -EEXIST; |
1202 | goto fail; |
1203 | } |
1204 | |
1205 | tm_vdev = kzalloc(size: sizeof(*tm_vdev), GFP_KERNEL); |
1206 | if (!tm_vdev) { |
1207 | ret = -ENOMEM; |
1208 | goto fail; |
1209 | } |
1210 | |
1211 | tm_vdev->vdev.id.device = vdev_id; |
1212 | tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops; |
1213 | tm_vdev->vdev.dev.parent = dev; |
1214 | tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release; |
1215 | tm_vdev->features = features; |
1216 | if (config) |
1217 | memcpy(&tm_vdev->config, config, size); |
1218 | |
1219 | if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) { |
1220 | dev_err(dev, "unable to allocate vring\n" ); |
1221 | ret = -ENOMEM; |
1222 | goto vdev_fail; |
1223 | } |
1224 | |
1225 | /* Allocate an output buffer for the console device. */ |
1226 | if (vdev_id == VIRTIO_ID_CONSOLE) |
1227 | tm_vdev->tx_buf.buf = devm_kmalloc(dev, |
1228 | MLXBF_TMFIFO_CON_TX_BUF_SIZE, |
1229 | GFP_KERNEL); |
1230 | fifo->vdev[vdev_id] = tm_vdev; |
1231 | |
1232 | /* Register the virtio device. */ |
1233 | ret = register_virtio_device(dev: &tm_vdev->vdev); |
1234 | reg_dev = tm_vdev; |
1235 | if (ret) { |
1236 | dev_err(dev, "register_virtio_device failed\n" ); |
1237 | goto vdev_fail; |
1238 | } |
1239 | |
1240 | mutex_unlock(lock: &fifo->lock); |
1241 | return 0; |
1242 | |
1243 | vdev_fail: |
1244 | mlxbf_tmfifo_free_vrings(fifo, tm_vdev); |
1245 | fifo->vdev[vdev_id] = NULL; |
1246 | if (reg_dev) |
1247 | put_device(dev: &tm_vdev->vdev.dev); |
1248 | else |
1249 | kfree(objp: tm_vdev); |
1250 | fail: |
1251 | mutex_unlock(lock: &fifo->lock); |
1252 | return ret; |
1253 | } |
1254 | |
1255 | /* Delete vdev for the FIFO. */ |
1256 | static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id) |
1257 | { |
1258 | struct mlxbf_tmfifo_vdev *tm_vdev; |
1259 | |
1260 | mutex_lock(&fifo->lock); |
1261 | |
1262 | /* Unregister vdev. */ |
1263 | tm_vdev = fifo->vdev[vdev_id]; |
1264 | if (tm_vdev) { |
1265 | unregister_virtio_device(dev: &tm_vdev->vdev); |
1266 | mlxbf_tmfifo_free_vrings(fifo, tm_vdev); |
1267 | fifo->vdev[vdev_id] = NULL; |
1268 | } |
1269 | |
1270 | mutex_unlock(lock: &fifo->lock); |
1271 | |
1272 | return 0; |
1273 | } |
1274 | |
1275 | /* Read the configured network MAC address from efi variable. */ |
1276 | static void mlxbf_tmfifo_get_cfg_mac(u8 *mac) |
1277 | { |
1278 | efi_guid_t guid = EFI_GLOBAL_VARIABLE_GUID; |
1279 | unsigned long size = ETH_ALEN; |
1280 | u8 buf[ETH_ALEN]; |
1281 | efi_status_t rc; |
1282 | |
1283 | rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf); |
1284 | if (rc == EFI_SUCCESS && size == ETH_ALEN) |
1285 | ether_addr_copy(dst: mac, src: buf); |
1286 | else |
1287 | ether_addr_copy(dst: mac, src: mlxbf_tmfifo_net_default_mac); |
1288 | } |
1289 | |
1290 | /* Set TmFifo thresolds which is used to trigger interrupts. */ |
1291 | static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo) |
1292 | { |
1293 | u64 ctl; |
1294 | |
1295 | /* Get Tx FIFO size and set the low/high watermark. */ |
1296 | ctl = readq(addr: fifo->tx.ctl); |
1297 | fifo->tx_fifo_size = |
1298 | FIELD_GET(MLXBF_TMFIFO_TX_CTL__MAX_ENTRIES_MASK, ctl); |
1299 | ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__LWM_MASK) | |
1300 | FIELD_PREP(MLXBF_TMFIFO_TX_CTL__LWM_MASK, |
1301 | fifo->tx_fifo_size / 2); |
1302 | ctl = (ctl & ~MLXBF_TMFIFO_TX_CTL__HWM_MASK) | |
1303 | FIELD_PREP(MLXBF_TMFIFO_TX_CTL__HWM_MASK, |
1304 | fifo->tx_fifo_size - 1); |
1305 | writeq(val: ctl, addr: fifo->tx.ctl); |
1306 | |
1307 | /* Get Rx FIFO size and set the low/high watermark. */ |
1308 | ctl = readq(addr: fifo->rx.ctl); |
1309 | fifo->rx_fifo_size = |
1310 | FIELD_GET(MLXBF_TMFIFO_RX_CTL__MAX_ENTRIES_MASK, ctl); |
1311 | ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__LWM_MASK) | |
1312 | FIELD_PREP(MLXBF_TMFIFO_RX_CTL__LWM_MASK, 0); |
1313 | ctl = (ctl & ~MLXBF_TMFIFO_RX_CTL__HWM_MASK) | |
1314 | FIELD_PREP(MLXBF_TMFIFO_RX_CTL__HWM_MASK, 1); |
1315 | writeq(val: ctl, addr: fifo->rx.ctl); |
1316 | } |
1317 | |
1318 | static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo) |
1319 | { |
1320 | int i; |
1321 | |
1322 | fifo->is_ready = false; |
1323 | del_timer_sync(timer: &fifo->timer); |
1324 | mlxbf_tmfifo_disable_irqs(fifo); |
1325 | cancel_work_sync(work: &fifo->work); |
1326 | for (i = 0; i < MLXBF_TMFIFO_VDEV_MAX; i++) |
1327 | mlxbf_tmfifo_delete_vdev(fifo, vdev_id: i); |
1328 | } |
1329 | |
1330 | /* Probe the TMFIFO. */ |
1331 | static int mlxbf_tmfifo_probe(struct platform_device *pdev) |
1332 | { |
1333 | struct virtio_net_config net_config; |
1334 | struct device *dev = &pdev->dev; |
1335 | struct mlxbf_tmfifo *fifo; |
1336 | u64 dev_id; |
1337 | int i, rc; |
1338 | |
1339 | rc = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), integer: &dev_id); |
1340 | if (rc) { |
1341 | dev_err(dev, "Cannot retrieve UID\n" ); |
1342 | return rc; |
1343 | } |
1344 | |
1345 | fifo = devm_kzalloc(dev, size: sizeof(*fifo), GFP_KERNEL); |
1346 | if (!fifo) |
1347 | return -ENOMEM; |
1348 | |
1349 | spin_lock_init(&fifo->spin_lock[0]); |
1350 | spin_lock_init(&fifo->spin_lock[1]); |
1351 | INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler); |
1352 | mutex_init(&fifo->lock); |
1353 | |
1354 | /* Get the resource of the Rx FIFO. */ |
1355 | fifo->res0 = devm_platform_ioremap_resource(pdev, index: 0); |
1356 | if (IS_ERR(ptr: fifo->res0)) |
1357 | return PTR_ERR(ptr: fifo->res0); |
1358 | |
1359 | /* Get the resource of the Tx FIFO. */ |
1360 | fifo->res1 = devm_platform_ioremap_resource(pdev, index: 1); |
1361 | if (IS_ERR(ptr: fifo->res1)) |
1362 | return PTR_ERR(ptr: fifo->res1); |
1363 | |
1364 | if (dev_id == TMFIFO_BF3_UID) { |
1365 | fifo->rx.ctl = fifo->res1 + MLXBF_TMFIFO_RX_CTL_BF3; |
1366 | fifo->rx.sts = fifo->res1 + MLXBF_TMFIFO_RX_STS_BF3; |
1367 | fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA_BF3; |
1368 | fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL_BF3; |
1369 | fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS_BF3; |
1370 | fifo->tx.data = fifo->res0 + MLXBF_TMFIFO_TX_DATA_BF3; |
1371 | } else { |
1372 | fifo->rx.ctl = fifo->res0 + MLXBF_TMFIFO_RX_CTL; |
1373 | fifo->rx.sts = fifo->res0 + MLXBF_TMFIFO_RX_STS; |
1374 | fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA; |
1375 | fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL; |
1376 | fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS; |
1377 | fifo->tx.data = fifo->res1 + MLXBF_TMFIFO_TX_DATA; |
1378 | } |
1379 | |
1380 | platform_set_drvdata(pdev, data: fifo); |
1381 | |
1382 | timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0); |
1383 | |
1384 | for (i = 0; i < MLXBF_TM_MAX_IRQ; i++) { |
1385 | fifo->irq_info[i].index = i; |
1386 | fifo->irq_info[i].fifo = fifo; |
1387 | fifo->irq_info[i].irq = platform_get_irq(pdev, i); |
1388 | rc = devm_request_irq(dev, irq: fifo->irq_info[i].irq, |
1389 | handler: mlxbf_tmfifo_irq_handler, irqflags: 0, |
1390 | devname: "tmfifo" , dev_id: &fifo->irq_info[i]); |
1391 | if (rc) { |
1392 | dev_err(dev, "devm_request_irq failed\n" ); |
1393 | fifo->irq_info[i].irq = 0; |
1394 | return rc; |
1395 | } |
1396 | } |
1397 | |
1398 | mlxbf_tmfifo_set_threshold(fifo); |
1399 | |
1400 | /* Create the console vdev. */ |
1401 | rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, features: 0, NULL, size: 0); |
1402 | if (rc) |
1403 | goto fail; |
1404 | |
1405 | /* Create the network vdev. */ |
1406 | memset(&net_config, 0, sizeof(net_config)); |
1407 | |
1408 | /* A legacy-only interface for now. */ |
1409 | net_config.mtu = __cpu_to_virtio16(little_endian: virtio_legacy_is_little_endian(), |
1410 | ETH_DATA_LEN); |
1411 | net_config.status = __cpu_to_virtio16(little_endian: virtio_legacy_is_little_endian(), |
1412 | VIRTIO_NET_S_LINK_UP); |
1413 | mlxbf_tmfifo_get_cfg_mac(mac: net_config.mac); |
1414 | rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET, |
1415 | MLXBF_TMFIFO_NET_FEATURES, config: &net_config, |
1416 | size: sizeof(net_config)); |
1417 | if (rc) |
1418 | goto fail; |
1419 | |
1420 | mod_timer(timer: &fifo->timer, expires: jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); |
1421 | |
1422 | /* Make all updates visible before setting the 'is_ready' flag. */ |
1423 | virtio_mb(weak_barriers: false); |
1424 | |
1425 | fifo->is_ready = true; |
1426 | return 0; |
1427 | |
1428 | fail: |
1429 | mlxbf_tmfifo_cleanup(fifo); |
1430 | return rc; |
1431 | } |
1432 | |
1433 | /* Device remove function. */ |
1434 | static void mlxbf_tmfifo_remove(struct platform_device *pdev) |
1435 | { |
1436 | struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev); |
1437 | |
1438 | mlxbf_tmfifo_cleanup(fifo); |
1439 | } |
1440 | |
1441 | static const struct acpi_device_id mlxbf_tmfifo_acpi_match[] = { |
1442 | { "MLNXBF01" , 0 }, |
1443 | {} |
1444 | }; |
1445 | MODULE_DEVICE_TABLE(acpi, mlxbf_tmfifo_acpi_match); |
1446 | |
1447 | static struct platform_driver mlxbf_tmfifo_driver = { |
1448 | .probe = mlxbf_tmfifo_probe, |
1449 | .remove_new = mlxbf_tmfifo_remove, |
1450 | .driver = { |
1451 | .name = "bf-tmfifo" , |
1452 | .acpi_match_table = mlxbf_tmfifo_acpi_match, |
1453 | }, |
1454 | }; |
1455 | |
1456 | module_platform_driver(mlxbf_tmfifo_driver); |
1457 | |
1458 | MODULE_DESCRIPTION("Mellanox BlueField SoC TmFifo Driver" ); |
1459 | MODULE_LICENSE("GPL v2" ); |
1460 | MODULE_AUTHOR("Mellanox Technologies" ); |
1461 | |