1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * SolidRun DPU driver for control plane |
4 | * |
5 | * Copyright (C) 2022-2023 SolidRun |
6 | * |
7 | * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> |
8 | * |
9 | */ |
10 | #include <linux/iopoll.h> |
11 | |
12 | #include "snet_vdpa.h" |
13 | |
14 | /* SNET DPU device ID */ |
15 | #define SNET_DEVICE_ID 0x1000 |
16 | /* SNET signature */ |
17 | #define SNET_SIGNATURE 0xD0D06363 |
18 | /* Max. config version that we can work with */ |
19 | #define SNET_CFG_VERSION 0x2 |
20 | /* Queue align */ |
21 | #define SNET_QUEUE_ALIGNMENT PAGE_SIZE |
22 | /* Kick value to notify that new data is available */ |
23 | #define SNET_KICK_VAL 0x1 |
24 | #define SNET_CONFIG_OFF 0x0 |
25 | /* How long we are willing to wait for a SNET device */ |
26 | #define SNET_DETECT_TIMEOUT 5000000 |
27 | /* How long should we wait for the DPU to read our config */ |
28 | #define SNET_READ_CFG_TIMEOUT 3000000 |
29 | /* Size of configs written to the DPU */ |
30 | #define SNET_GENERAL_CFG_LEN 36 |
31 | #define SNET_GENERAL_CFG_VQ_LEN 40 |
32 | |
33 | static struct snet *vdpa_to_snet(struct vdpa_device *vdpa) |
34 | { |
35 | return container_of(vdpa, struct snet, vdpa); |
36 | } |
37 | |
38 | static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data) |
39 | { |
40 | struct snet *snet = data; |
41 | /* Call callback if any */ |
42 | if (likely(snet->cb.callback)) |
43 | return snet->cb.callback(snet->cb.private); |
44 | |
45 | return IRQ_HANDLED; |
46 | } |
47 | |
48 | static irqreturn_t snet_vq_irq_hndlr(int irq, void *data) |
49 | { |
50 | struct snet_vq *vq = data; |
51 | /* Call callback if any */ |
52 | if (likely(vq->cb.callback)) |
53 | return vq->cb.callback(vq->cb.private); |
54 | |
55 | return IRQ_HANDLED; |
56 | } |
57 | |
58 | static void snet_free_irqs(struct snet *snet) |
59 | { |
60 | struct psnet *psnet = snet->psnet; |
61 | struct pci_dev *pdev; |
62 | u32 i; |
63 | |
64 | /* Which Device allcoated the IRQs? */ |
65 | if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) |
66 | pdev = snet->pdev->physfn; |
67 | else |
68 | pdev = snet->pdev; |
69 | |
70 | /* Free config's IRQ */ |
71 | if (snet->cfg_irq != -1) { |
72 | devm_free_irq(dev: &pdev->dev, irq: snet->cfg_irq, dev_id: snet); |
73 | snet->cfg_irq = -1; |
74 | } |
75 | /* Free VQ IRQs */ |
76 | for (i = 0; i < snet->cfg->vq_num; i++) { |
77 | if (snet->vqs[i] && snet->vqs[i]->irq != -1) { |
78 | devm_free_irq(dev: &pdev->dev, irq: snet->vqs[i]->irq, dev_id: snet->vqs[i]); |
79 | snet->vqs[i]->irq = -1; |
80 | } |
81 | } |
82 | |
83 | /* IRQ vectors are freed when the pci remove callback is called */ |
84 | } |
85 | |
86 | static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area, |
87 | u64 driver_area, u64 device_area) |
88 | { |
89 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
90 | /* save received parameters in vqueue sturct */ |
91 | snet->vqs[idx]->desc_area = desc_area; |
92 | snet->vqs[idx]->driver_area = driver_area; |
93 | snet->vqs[idx]->device_area = device_area; |
94 | |
95 | return 0; |
96 | } |
97 | |
98 | static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) |
99 | { |
100 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
101 | /* save num in vqueue */ |
102 | snet->vqs[idx]->num = num; |
103 | } |
104 | |
105 | static void snet_kick_vq(struct vdpa_device *vdev, u16 idx) |
106 | { |
107 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
108 | /* not ready - ignore */ |
109 | if (unlikely(!snet->vqs[idx]->ready)) |
110 | return; |
111 | |
112 | iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); |
113 | } |
114 | |
115 | static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data) |
116 | { |
117 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
118 | u16 idx = data & 0xFFFF; |
119 | |
120 | /* not ready - ignore */ |
121 | if (unlikely(!snet->vqs[idx]->ready)) |
122 | return; |
123 | |
124 | iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); |
125 | } |
126 | |
127 | static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb) |
128 | { |
129 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
130 | |
131 | snet->vqs[idx]->cb.callback = cb->callback; |
132 | snet->vqs[idx]->cb.private = cb->private; |
133 | } |
134 | |
135 | static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready) |
136 | { |
137 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
138 | |
139 | snet->vqs[idx]->ready = ready; |
140 | } |
141 | |
142 | static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx) |
143 | { |
144 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
145 | |
146 | return snet->vqs[idx]->ready; |
147 | } |
148 | |
149 | static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state) |
150 | { |
151 | if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) { |
152 | const struct vdpa_vq_state_packed *p = &state->packed; |
153 | |
154 | if (p->last_avail_counter == 1 && p->last_used_counter == 1 && |
155 | p->last_avail_idx == 0 && p->last_used_idx == 0) |
156 | return true; |
157 | } else { |
158 | const struct vdpa_vq_state_split *s = &state->split; |
159 | |
160 | if (s->avail_index == 0) |
161 | return true; |
162 | } |
163 | |
164 | return false; |
165 | } |
166 | |
167 | static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state) |
168 | { |
169 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
170 | |
171 | /* We can set any state for config version 2+ */ |
172 | if (SNET_CFG_VER(snet, 2)) { |
173 | memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state)); |
174 | return 0; |
175 | } |
176 | |
177 | /* Older config - we can't set the VQ state. |
178 | * Return 0 only if this is the initial state we use in the DPU. |
179 | */ |
180 | if (snet_vq_state_is_initial(snet, state)) |
181 | return 0; |
182 | |
183 | return -EOPNOTSUPP; |
184 | } |
185 | |
186 | static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state) |
187 | { |
188 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
189 | |
190 | return snet_read_vq_state(snet, idx, state); |
191 | } |
192 | |
193 | static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx) |
194 | { |
195 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
196 | |
197 | return snet->vqs[idx]->irq; |
198 | } |
199 | |
200 | static u32 snet_get_vq_align(struct vdpa_device *vdev) |
201 | { |
202 | return (u32)SNET_QUEUE_ALIGNMENT; |
203 | } |
204 | |
205 | static int snet_reset_dev(struct snet *snet) |
206 | { |
207 | struct pci_dev *pdev = snet->pdev; |
208 | int ret = 0; |
209 | u32 i; |
210 | |
211 | /* If status is 0, nothing to do */ |
212 | if (!snet->status) |
213 | return 0; |
214 | |
215 | /* If DPU started, destroy it */ |
216 | if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) |
217 | ret = snet_destroy_dev(snet); |
218 | |
219 | /* Clear VQs */ |
220 | for (i = 0; i < snet->cfg->vq_num; i++) { |
221 | if (!snet->vqs[i]) |
222 | continue; |
223 | snet->vqs[i]->cb.callback = NULL; |
224 | snet->vqs[i]->cb.private = NULL; |
225 | snet->vqs[i]->desc_area = 0; |
226 | snet->vqs[i]->device_area = 0; |
227 | snet->vqs[i]->driver_area = 0; |
228 | snet->vqs[i]->ready = false; |
229 | } |
230 | |
231 | /* Clear config callback */ |
232 | snet->cb.callback = NULL; |
233 | snet->cb.private = NULL; |
234 | /* Free IRQs */ |
235 | snet_free_irqs(snet); |
236 | /* Reset status */ |
237 | snet->status = 0; |
238 | snet->dpu_ready = false; |
239 | |
240 | if (ret) |
241 | SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n" , snet->sid, ret); |
242 | else |
243 | SNET_DBG(pdev, "Reset SNET[%u] device\n" , snet->sid); |
244 | |
245 | return 0; |
246 | } |
247 | |
248 | static int snet_reset(struct vdpa_device *vdev) |
249 | { |
250 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
251 | |
252 | return snet_reset_dev(snet); |
253 | } |
254 | |
255 | static size_t snet_get_config_size(struct vdpa_device *vdev) |
256 | { |
257 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
258 | |
259 | return (size_t)snet->cfg->cfg_size; |
260 | } |
261 | |
262 | static u64 snet_get_features(struct vdpa_device *vdev) |
263 | { |
264 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
265 | |
266 | return snet->cfg->features; |
267 | } |
268 | |
269 | static int snet_set_drv_features(struct vdpa_device *vdev, u64 features) |
270 | { |
271 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
272 | |
273 | snet->negotiated_features = snet->cfg->features & features; |
274 | return 0; |
275 | } |
276 | |
277 | static u64 snet_get_drv_features(struct vdpa_device *vdev) |
278 | { |
279 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
280 | |
281 | return snet->negotiated_features; |
282 | } |
283 | |
284 | static u16 snet_get_vq_num_max(struct vdpa_device *vdev) |
285 | { |
286 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
287 | |
288 | return (u16)snet->cfg->vq_size; |
289 | } |
290 | |
291 | static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb) |
292 | { |
293 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
294 | |
295 | snet->cb.callback = cb->callback; |
296 | snet->cb.private = cb->private; |
297 | } |
298 | |
299 | static u32 snet_get_device_id(struct vdpa_device *vdev) |
300 | { |
301 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
302 | |
303 | return snet->cfg->virtio_id; |
304 | } |
305 | |
306 | static u32 snet_get_vendor_id(struct vdpa_device *vdev) |
307 | { |
308 | return (u32)PCI_VENDOR_ID_SOLIDRUN; |
309 | } |
310 | |
311 | static u8 snet_get_status(struct vdpa_device *vdev) |
312 | { |
313 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
314 | |
315 | return snet->status; |
316 | } |
317 | |
318 | static int snet_write_conf(struct snet *snet) |
319 | { |
320 | u32 off, i, tmp; |
321 | int ret; |
322 | |
323 | /* No need to write the config twice */ |
324 | if (snet->dpu_ready) |
325 | return true; |
326 | |
327 | /* Snet data : |
328 | * |
329 | * General data: SNET_GENERAL_CFG_LEN bytes long |
330 | * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24 |
331 | * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD | |
332 | * |
333 | * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long |
334 | * 0 0x4 0x8 |
335 | * | VQ SID AND QUEUE SIZE | IRQ Index | |
336 | * | DESC AREA | |
337 | * | DEVICE AREA | |
338 | * | DRIVER AREA | |
339 | * | VQ STATE (CFG 2+) | RSVD | |
340 | * |
341 | * Magic number should be written last, this is the DPU indication that the data is ready |
342 | */ |
343 | |
344 | /* Init offset */ |
345 | off = snet->psnet->cfg.host_cfg_off; |
346 | |
347 | /* Ignore magic number for now */ |
348 | off += 4; |
349 | snet_write32(snet, off, val: snet->psnet->negotiated_cfg_ver); |
350 | off += 4; |
351 | snet_write32(snet, off, val: snet->sid); |
352 | off += 4; |
353 | snet_write32(snet, off, val: snet->cfg->vq_num); |
354 | off += 4; |
355 | snet_write32(snet, off, val: snet->cfg_irq_idx); |
356 | off += 4; |
357 | snet_write64(snet, off, val: snet->negotiated_features); |
358 | off += 8; |
359 | /* Ignore reserved */ |
360 | off += 8; |
361 | /* Write VQs */ |
362 | for (i = 0 ; i < snet->cfg->vq_num ; i++) { |
363 | tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); |
364 | snet_write32(snet, off, val: tmp); |
365 | off += 4; |
366 | snet_write32(snet, off, val: snet->vqs[i]->irq_idx); |
367 | off += 4; |
368 | snet_write64(snet, off, val: snet->vqs[i]->desc_area); |
369 | off += 8; |
370 | snet_write64(snet, off, val: snet->vqs[i]->device_area); |
371 | off += 8; |
372 | snet_write64(snet, off, val: snet->vqs[i]->driver_area); |
373 | off += 8; |
374 | /* Write VQ state if config version is 2+ */ |
375 | if (SNET_CFG_VER(snet, 2)) |
376 | snet_write32(snet, off, val: *(u32 *)&snet->vqs[i]->vq_state); |
377 | off += 4; |
378 | |
379 | /* Ignore reserved */ |
380 | off += 4; |
381 | } |
382 | |
383 | /* Write magic number - data is ready */ |
384 | snet_write32(snet, off: snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); |
385 | |
386 | /* The DPU will ACK the config by clearing the signature */ |
387 | ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, |
388 | tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT); |
389 | if (ret) { |
390 | SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n" ); |
391 | return false; |
392 | } |
393 | |
394 | /* set DPU flag */ |
395 | snet->dpu_ready = true; |
396 | |
397 | return true; |
398 | } |
399 | |
400 | static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) |
401 | { |
402 | int ret, i, irq; |
403 | |
404 | /* Request config IRQ */ |
405 | irq = pci_irq_vector(dev: pdev, nr: snet->cfg_irq_idx); |
406 | ret = devm_request_irq(dev: &pdev->dev, irq, handler: snet_cfg_irq_hndlr, irqflags: 0, |
407 | devname: snet->cfg_irq_name, dev_id: snet); |
408 | if (ret) { |
409 | SNET_ERR(pdev, "Failed to request IRQ\n" ); |
410 | return ret; |
411 | } |
412 | snet->cfg_irq = irq; |
413 | |
414 | /* Request IRQ for every VQ */ |
415 | for (i = 0; i < snet->cfg->vq_num; i++) { |
416 | irq = pci_irq_vector(dev: pdev, nr: snet->vqs[i]->irq_idx); |
417 | ret = devm_request_irq(dev: &pdev->dev, irq, handler: snet_vq_irq_hndlr, irqflags: 0, |
418 | devname: snet->vqs[i]->irq_name, dev_id: snet->vqs[i]); |
419 | if (ret) { |
420 | SNET_ERR(pdev, "Failed to request IRQ\n" ); |
421 | return ret; |
422 | } |
423 | snet->vqs[i]->irq = irq; |
424 | } |
425 | return 0; |
426 | } |
427 | |
428 | static void snet_set_status(struct vdpa_device *vdev, u8 status) |
429 | { |
430 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
431 | struct psnet *psnet = snet->psnet; |
432 | struct pci_dev *pdev = snet->pdev; |
433 | int ret; |
434 | bool pf_irqs; |
435 | |
436 | if (status == snet->status) |
437 | return; |
438 | |
439 | if ((status & VIRTIO_CONFIG_S_DRIVER_OK) && |
440 | !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { |
441 | /* Request IRQs */ |
442 | pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); |
443 | ret = snet_request_irqs(pdev: pf_irqs ? pdev->physfn : pdev, snet); |
444 | if (ret) |
445 | goto set_err; |
446 | |
447 | /* Write config to the DPU */ |
448 | if (snet_write_conf(snet)) { |
449 | SNET_INFO(pdev, "Create SNET[%u] device\n" , snet->sid); |
450 | } else { |
451 | snet_free_irqs(snet); |
452 | goto set_err; |
453 | } |
454 | } |
455 | |
456 | /* Save the new status */ |
457 | snet->status = status; |
458 | return; |
459 | |
460 | set_err: |
461 | snet->status |= VIRTIO_CONFIG_S_FAILED; |
462 | } |
463 | |
464 | static void snet_get_config(struct vdpa_device *vdev, unsigned int offset, |
465 | void *buf, unsigned int len) |
466 | { |
467 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
468 | void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; |
469 | u8 *buf_ptr = buf; |
470 | u32 i; |
471 | |
472 | /* check for offset error */ |
473 | if (offset + len > snet->cfg->cfg_size) |
474 | return; |
475 | |
476 | /* Write into buffer */ |
477 | for (i = 0; i < len; i++) |
478 | *buf_ptr++ = ioread8(cfg_ptr + i); |
479 | } |
480 | |
481 | static void snet_set_config(struct vdpa_device *vdev, unsigned int offset, |
482 | const void *buf, unsigned int len) |
483 | { |
484 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
485 | void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; |
486 | const u8 *buf_ptr = buf; |
487 | u32 i; |
488 | |
489 | /* check for offset error */ |
490 | if (offset + len > snet->cfg->cfg_size) |
491 | return; |
492 | |
493 | /* Write into PCI BAR */ |
494 | for (i = 0; i < len; i++) |
495 | iowrite8(*buf_ptr++, cfg_ptr + i); |
496 | } |
497 | |
498 | static int snet_suspend(struct vdpa_device *vdev) |
499 | { |
500 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
501 | int ret; |
502 | |
503 | ret = snet_suspend_dev(snet); |
504 | if (ret) |
505 | SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n" , snet->sid, ret); |
506 | else |
507 | SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n" , snet->sid); |
508 | |
509 | return ret; |
510 | } |
511 | |
512 | static int snet_resume(struct vdpa_device *vdev) |
513 | { |
514 | struct snet *snet = vdpa_to_snet(vdpa: vdev); |
515 | int ret; |
516 | |
517 | ret = snet_resume_dev(snet); |
518 | if (ret) |
519 | SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n" , snet->sid, ret); |
520 | else |
521 | SNET_DBG(snet->pdev, "Resume SNET[%u] device\n" , snet->sid); |
522 | |
523 | return ret; |
524 | } |
525 | |
526 | static const struct vdpa_config_ops snet_config_ops = { |
527 | .set_vq_address = snet_set_vq_address, |
528 | .set_vq_num = snet_set_vq_num, |
529 | .kick_vq = snet_kick_vq, |
530 | .kick_vq_with_data = snet_kick_vq_with_data, |
531 | .set_vq_cb = snet_set_vq_cb, |
532 | .set_vq_ready = snet_set_vq_ready, |
533 | .get_vq_ready = snet_get_vq_ready, |
534 | .set_vq_state = snet_set_vq_state, |
535 | .get_vq_state = snet_get_vq_state, |
536 | .get_vq_irq = snet_get_vq_irq, |
537 | .get_vq_align = snet_get_vq_align, |
538 | .reset = snet_reset, |
539 | .get_config_size = snet_get_config_size, |
540 | .get_device_features = snet_get_features, |
541 | .set_driver_features = snet_set_drv_features, |
542 | .get_driver_features = snet_get_drv_features, |
543 | .get_vq_num_min = snet_get_vq_num_max, |
544 | .get_vq_num_max = snet_get_vq_num_max, |
545 | .set_config_cb = snet_set_config_cb, |
546 | .get_device_id = snet_get_device_id, |
547 | .get_vendor_id = snet_get_vendor_id, |
548 | .get_status = snet_get_status, |
549 | .set_status = snet_set_status, |
550 | .get_config = snet_get_config, |
551 | .set_config = snet_set_config, |
552 | .suspend = snet_suspend, |
553 | .resume = snet_resume, |
554 | }; |
555 | |
556 | static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet) |
557 | { |
558 | char name[50]; |
559 | int ret, i, mask = 0; |
560 | /* We don't know which BAR will be used to communicate.. |
561 | * We will map every bar with len > 0. |
562 | * |
563 | * Later, we will discover the BAR and unmap all other BARs. |
564 | */ |
565 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
566 | if (pci_resource_len(pdev, i)) |
567 | mask |= (1 << i); |
568 | } |
569 | |
570 | /* No BAR can be used.. */ |
571 | if (!mask) { |
572 | SNET_ERR(pdev, "Failed to find a PCI BAR\n" ); |
573 | return -ENODEV; |
574 | } |
575 | |
576 | snprintf(buf: name, size: sizeof(name), fmt: "psnet[%s]-bars" , pci_name(pdev)); |
577 | ret = pcim_iomap_regions(pdev, mask, name); |
578 | if (ret) { |
579 | SNET_ERR(pdev, "Failed to request and map PCI BARs\n" ); |
580 | return ret; |
581 | } |
582 | |
583 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
584 | if (mask & (1 << i)) |
585 | psnet->bars[i] = pcim_iomap_table(pdev)[i]; |
586 | } |
587 | |
588 | return 0; |
589 | } |
590 | |
591 | static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet) |
592 | { |
593 | char name[50]; |
594 | int ret; |
595 | |
596 | snprintf(buf: name, size: sizeof(name), fmt: "snet[%s]-bar" , pci_name(pdev)); |
597 | /* Request and map BAR */ |
598 | ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name); |
599 | if (ret) { |
600 | SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n" ); |
601 | return ret; |
602 | } |
603 | |
604 | snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar]; |
605 | |
606 | return 0; |
607 | } |
608 | |
609 | static void snet_free_cfg(struct snet_cfg *cfg) |
610 | { |
611 | u32 i; |
612 | |
613 | if (!cfg->devs) |
614 | return; |
615 | |
616 | /* Free devices */ |
617 | for (i = 0; i < cfg->devices_num; i++) { |
618 | if (!cfg->devs[i]) |
619 | break; |
620 | |
621 | kfree(objp: cfg->devs[i]); |
622 | } |
623 | /* Free pointers to devices */ |
624 | kfree(objp: cfg->devs); |
625 | } |
626 | |
627 | /* Detect which BAR is used for communication with the device. */ |
628 | static int psnet_detect_bar(struct psnet *psnet, u32 off) |
629 | { |
630 | unsigned long exit_time; |
631 | int i; |
632 | |
633 | exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT); |
634 | |
635 | /* SNET DPU will write SNET's signature when the config is ready. */ |
636 | while (time_before(jiffies, exit_time)) { |
637 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
638 | /* Is this BAR mapped? */ |
639 | if (!psnet->bars[i]) |
640 | continue; |
641 | |
642 | if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) |
643 | return i; |
644 | } |
645 | usleep_range(min: 1000, max: 10000); |
646 | } |
647 | |
648 | return -ENODEV; |
649 | } |
650 | |
651 | static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet) |
652 | { |
653 | int i, mask = 0; |
654 | |
655 | for (i = 0; i < PCI_STD_NUM_BARS; i++) { |
656 | if (psnet->bars[i] && i != psnet->barno) |
657 | mask |= (1 << i); |
658 | } |
659 | |
660 | if (mask) |
661 | pcim_iounmap_regions(pdev, mask); |
662 | } |
663 | |
664 | /* Read SNET config from PCI BAR */ |
665 | static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet) |
666 | { |
667 | struct snet_cfg *cfg = &psnet->cfg; |
668 | u32 i, off; |
669 | int barno; |
670 | |
671 | /* Move to where the config starts */ |
672 | off = SNET_CONFIG_OFF; |
673 | |
674 | /* Find BAR used for communication */ |
675 | barno = psnet_detect_bar(psnet, off); |
676 | if (barno < 0) { |
677 | SNET_ERR(pdev, "SNET config is not ready.\n" ); |
678 | return barno; |
679 | } |
680 | |
681 | /* Save used BAR number and unmap all other BARs */ |
682 | psnet->barno = barno; |
683 | SNET_DBG(pdev, "Using BAR number %d\n" , barno); |
684 | |
685 | psnet_unmap_unused_bars(pdev, psnet); |
686 | |
687 | /* load config from BAR */ |
688 | cfg->key = psnet_read32(psnet, off); |
689 | off += 4; |
690 | cfg->cfg_size = psnet_read32(psnet, off); |
691 | off += 4; |
692 | cfg->cfg_ver = psnet_read32(psnet, off); |
693 | off += 4; |
694 | /* The negotiated config version is the lower one between this driver's config |
695 | * and the DPU's. |
696 | */ |
697 | psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); |
698 | SNET_DBG(pdev, "SNET config version %u\n" , psnet->negotiated_cfg_ver); |
699 | |
700 | cfg->vf_num = psnet_read32(psnet, off); |
701 | off += 4; |
702 | cfg->vf_bar = psnet_read32(psnet, off); |
703 | off += 4; |
704 | cfg->host_cfg_off = psnet_read32(psnet, off); |
705 | off += 4; |
706 | cfg->max_size_host_cfg = psnet_read32(psnet, off); |
707 | off += 4; |
708 | cfg->virtio_cfg_off = psnet_read32(psnet, off); |
709 | off += 4; |
710 | cfg->kick_off = psnet_read32(psnet, off); |
711 | off += 4; |
712 | cfg->hwmon_off = psnet_read32(psnet, off); |
713 | off += 4; |
714 | cfg->ctrl_off = psnet_read32(psnet, off); |
715 | off += 4; |
716 | cfg->flags = psnet_read32(psnet, off); |
717 | off += 4; |
718 | /* Ignore Reserved */ |
719 | off += sizeof(cfg->rsvd); |
720 | |
721 | cfg->devices_num = psnet_read32(psnet, off); |
722 | off += 4; |
723 | /* Allocate memory to hold pointer to the devices */ |
724 | cfg->devs = kcalloc(n: cfg->devices_num, size: sizeof(void *), GFP_KERNEL); |
725 | if (!cfg->devs) |
726 | return -ENOMEM; |
727 | |
728 | /* Load device configuration from BAR */ |
729 | for (i = 0; i < cfg->devices_num; i++) { |
730 | cfg->devs[i] = kzalloc(size: sizeof(*cfg->devs[i]), GFP_KERNEL); |
731 | if (!cfg->devs[i]) { |
732 | snet_free_cfg(cfg); |
733 | return -ENOMEM; |
734 | } |
735 | /* Read device config */ |
736 | cfg->devs[i]->virtio_id = psnet_read32(psnet, off); |
737 | off += 4; |
738 | cfg->devs[i]->vq_num = psnet_read32(psnet, off); |
739 | off += 4; |
740 | cfg->devs[i]->vq_size = psnet_read32(psnet, off); |
741 | off += 4; |
742 | cfg->devs[i]->vfid = psnet_read32(psnet, off); |
743 | off += 4; |
744 | cfg->devs[i]->features = psnet_read64(psnet, off); |
745 | off += 8; |
746 | /* Ignore Reserved */ |
747 | off += sizeof(cfg->devs[i]->rsvd); |
748 | |
749 | cfg->devs[i]->cfg_size = psnet_read32(psnet, off); |
750 | off += 4; |
751 | |
752 | /* Is the config witten to the DPU going to be too big? */ |
753 | if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > |
754 | cfg->max_size_host_cfg) { |
755 | SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n" ); |
756 | snet_free_cfg(cfg); |
757 | return -EINVAL; |
758 | } |
759 | } |
760 | return 0; |
761 | } |
762 | |
763 | static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet) |
764 | { |
765 | int ret = 0; |
766 | u32 i, irq_num = 0; |
767 | |
768 | /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */ |
769 | for (i = 0; i < psnet->cfg.devices_num; i++) |
770 | irq_num += psnet->cfg.devs[i]->vq_num + 1; |
771 | |
772 | ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: irq_num, max_vecs: irq_num, PCI_IRQ_MSIX); |
773 | if (ret != irq_num) { |
774 | SNET_ERR(pdev, "Failed to allocate IRQ vectors\n" ); |
775 | return ret; |
776 | } |
777 | SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n" , irq_num); |
778 | |
779 | return 0; |
780 | } |
781 | |
782 | static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg) |
783 | { |
784 | int ret = 0; |
785 | u32 irq_num; |
786 | |
787 | /* We want 1 IRQ for every VQ + 1 for config change events */ |
788 | irq_num = snet_cfg->vq_num + 1; |
789 | |
790 | ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: irq_num, max_vecs: irq_num, PCI_IRQ_MSIX); |
791 | if (ret <= 0) { |
792 | SNET_ERR(pdev, "Failed to allocate IRQ vectors\n" ); |
793 | return ret; |
794 | } |
795 | |
796 | return 0; |
797 | } |
798 | |
799 | static void snet_free_vqs(struct snet *snet) |
800 | { |
801 | u32 i; |
802 | |
803 | if (!snet->vqs) |
804 | return; |
805 | |
806 | for (i = 0 ; i < snet->cfg->vq_num ; i++) { |
807 | if (!snet->vqs[i]) |
808 | break; |
809 | |
810 | kfree(objp: snet->vqs[i]); |
811 | } |
812 | kfree(objp: snet->vqs); |
813 | } |
814 | |
815 | static int snet_build_vqs(struct snet *snet) |
816 | { |
817 | u32 i; |
818 | /* Allocate the VQ pointers array */ |
819 | snet->vqs = kcalloc(n: snet->cfg->vq_num, size: sizeof(void *), GFP_KERNEL); |
820 | if (!snet->vqs) |
821 | return -ENOMEM; |
822 | |
823 | /* Allocate the VQs */ |
824 | for (i = 0; i < snet->cfg->vq_num; i++) { |
825 | snet->vqs[i] = kzalloc(size: sizeof(*snet->vqs[i]), GFP_KERNEL); |
826 | if (!snet->vqs[i]) { |
827 | snet_free_vqs(snet); |
828 | return -ENOMEM; |
829 | } |
830 | /* Reset IRQ num */ |
831 | snet->vqs[i]->irq = -1; |
832 | /* VQ serial ID */ |
833 | snet->vqs[i]->sid = i; |
834 | /* Kick address - every VQ gets 4B */ |
835 | snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + |
836 | snet->vqs[i]->sid * 4; |
837 | /* Clear kick address for this VQ */ |
838 | iowrite32(0, snet->vqs[i]->kick_ptr); |
839 | } |
840 | return 0; |
841 | } |
842 | |
843 | static int psnet_get_next_irq_num(struct psnet *psnet) |
844 | { |
845 | int irq; |
846 | |
847 | spin_lock(lock: &psnet->lock); |
848 | irq = psnet->next_irq++; |
849 | spin_unlock(lock: &psnet->lock); |
850 | |
851 | return irq; |
852 | } |
853 | |
854 | static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet) |
855 | { |
856 | struct psnet *psnet = snet->psnet; |
857 | int i; |
858 | |
859 | /* one IRQ for every VQ, and one for config changes */ |
860 | snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); |
861 | snprintf(buf: snet->cfg_irq_name, SNET_NAME_SIZE, fmt: "snet[%s]-cfg[%d]" , |
862 | pci_name(pdev), snet->cfg_irq_idx); |
863 | |
864 | for (i = 0; i < snet->cfg->vq_num; i++) { |
865 | /* Get next free IRQ ID */ |
866 | snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); |
867 | /* Write IRQ name */ |
868 | snprintf(buf: snet->vqs[i]->irq_name, SNET_NAME_SIZE, fmt: "snet[%s]-vq[%d]" , |
869 | pci_name(pdev), snet->vqs[i]->irq_idx); |
870 | } |
871 | } |
872 | |
873 | /* Find a device config based on virtual function id */ |
874 | static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid) |
875 | { |
876 | u32 i; |
877 | |
878 | for (i = 0; i < cfg->devices_num; i++) { |
879 | if (cfg->devs[i]->vfid == vfid) |
880 | return cfg->devs[i]; |
881 | } |
882 | /* Oppss.. no config found.. */ |
883 | return NULL; |
884 | } |
885 | |
886 | /* Probe function for a physical PCI function */ |
887 | static int snet_vdpa_probe_pf(struct pci_dev *pdev) |
888 | { |
889 | struct psnet *psnet; |
890 | int ret = 0; |
891 | bool pf_irqs = false; |
892 | |
893 | ret = pcim_enable_device(pdev); |
894 | if (ret) { |
895 | SNET_ERR(pdev, "Failed to enable PCI device\n" ); |
896 | return ret; |
897 | } |
898 | |
899 | /* Allocate a PCI physical function device */ |
900 | psnet = kzalloc(size: sizeof(*psnet), GFP_KERNEL); |
901 | if (!psnet) |
902 | return -ENOMEM; |
903 | |
904 | /* Init PSNET spinlock */ |
905 | spin_lock_init(&psnet->lock); |
906 | |
907 | pci_set_master(dev: pdev); |
908 | pci_set_drvdata(pdev, data: psnet); |
909 | |
910 | /* Open SNET MAIN BAR */ |
911 | ret = psnet_open_pf_bar(pdev, psnet); |
912 | if (ret) |
913 | goto free_psnet; |
914 | |
915 | /* Try to read SNET's config from PCI BAR */ |
916 | ret = psnet_read_cfg(pdev, psnet); |
917 | if (ret) |
918 | goto free_psnet; |
919 | |
920 | /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use |
921 | * PF MSI-X vectors |
922 | */ |
923 | pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); |
924 | |
925 | if (pf_irqs) { |
926 | ret = psnet_alloc_irq_vector(pdev, psnet); |
927 | if (ret) |
928 | goto free_cfg; |
929 | } |
930 | |
931 | SNET_DBG(pdev, "Enable %u virtual functions\n" , psnet->cfg.vf_num); |
932 | ret = pci_enable_sriov(dev: pdev, nr_virtfn: psnet->cfg.vf_num); |
933 | if (ret) { |
934 | SNET_ERR(pdev, "Failed to enable SR-IOV\n" ); |
935 | goto free_irq; |
936 | } |
937 | |
938 | /* Create HW monitor device */ |
939 | if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) { |
940 | #if IS_ENABLED(CONFIG_HWMON) |
941 | psnet_create_hwmon(pdev); |
942 | #else |
943 | SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n" ); |
944 | #endif |
945 | } |
946 | |
947 | return 0; |
948 | |
949 | free_irq: |
950 | if (pf_irqs) |
951 | pci_free_irq_vectors(dev: pdev); |
952 | free_cfg: |
953 | snet_free_cfg(cfg: &psnet->cfg); |
954 | free_psnet: |
955 | kfree(objp: psnet); |
956 | return ret; |
957 | } |
958 | |
959 | /* Probe function for a virtual PCI function */ |
960 | static int snet_vdpa_probe_vf(struct pci_dev *pdev) |
961 | { |
962 | struct pci_dev *pdev_pf = pdev->physfn; |
963 | struct psnet *psnet = pci_get_drvdata(pdev: pdev_pf); |
964 | struct snet_dev_cfg *dev_cfg; |
965 | struct snet *snet; |
966 | u32 vfid; |
967 | int ret; |
968 | bool pf_irqs = false; |
969 | |
970 | /* Get virtual function id. |
971 | * (the DPU counts the VFs from 1) |
972 | */ |
973 | ret = pci_iov_vf_id(dev: pdev); |
974 | if (ret < 0) { |
975 | SNET_ERR(pdev, "Failed to find a VF id\n" ); |
976 | return ret; |
977 | } |
978 | vfid = ret + 1; |
979 | |
980 | /* Find the snet_dev_cfg based on vfid */ |
981 | dev_cfg = snet_find_dev_cfg(cfg: &psnet->cfg, vfid); |
982 | if (!dev_cfg) { |
983 | SNET_WARN(pdev, "Failed to find a VF config..\n" ); |
984 | return -ENODEV; |
985 | } |
986 | |
987 | /* Which PCI device should allocate the IRQs? |
988 | * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs |
989 | */ |
990 | pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF); |
991 | |
992 | ret = pcim_enable_device(pdev); |
993 | if (ret) { |
994 | SNET_ERR(pdev, "Failed to enable PCI VF device\n" ); |
995 | return ret; |
996 | } |
997 | |
998 | /* Request for MSI-X IRQs */ |
999 | if (!pf_irqs) { |
1000 | ret = snet_alloc_irq_vector(pdev, snet_cfg: dev_cfg); |
1001 | if (ret) |
1002 | return ret; |
1003 | } |
1004 | |
1005 | /* Allocate vdpa device */ |
1006 | snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, |
1007 | false); |
1008 | if (!snet) { |
1009 | SNET_ERR(pdev, "Failed to allocate a vdpa device\n" ); |
1010 | ret = -ENOMEM; |
1011 | goto free_irqs; |
1012 | } |
1013 | |
1014 | /* Init control mutex and spinlock */ |
1015 | mutex_init(&snet->ctrl_lock); |
1016 | spin_lock_init(&snet->ctrl_spinlock); |
1017 | |
1018 | /* Save pci device pointer */ |
1019 | snet->pdev = pdev; |
1020 | snet->psnet = psnet; |
1021 | snet->cfg = dev_cfg; |
1022 | snet->dpu_ready = false; |
1023 | snet->sid = vfid; |
1024 | /* Reset IRQ value */ |
1025 | snet->cfg_irq = -1; |
1026 | |
1027 | ret = snet_open_vf_bar(pdev, snet); |
1028 | if (ret) |
1029 | goto put_device; |
1030 | |
1031 | /* Create a VirtIO config pointer */ |
1032 | snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; |
1033 | |
1034 | /* Clear control registers */ |
1035 | snet_ctrl_clear(snet); |
1036 | |
1037 | pci_set_master(dev: pdev); |
1038 | pci_set_drvdata(pdev, data: snet); |
1039 | |
1040 | ret = snet_build_vqs(snet); |
1041 | if (ret) |
1042 | goto put_device; |
1043 | |
1044 | /* Reserve IRQ indexes, |
1045 | * The IRQs may be requested and freed multiple times, |
1046 | * but the indexes won't change. |
1047 | */ |
1048 | snet_reserve_irq_idx(pdev: pf_irqs ? pdev_pf : pdev, snet); |
1049 | |
1050 | /*set DMA device*/ |
1051 | snet->vdpa.dma_dev = &pdev->dev; |
1052 | |
1053 | /* Register VDPA device */ |
1054 | ret = vdpa_register_device(vdev: &snet->vdpa, nvqs: snet->cfg->vq_num); |
1055 | if (ret) { |
1056 | SNET_ERR(pdev, "Failed to register vdpa device\n" ); |
1057 | goto free_vqs; |
1058 | } |
1059 | |
1060 | return 0; |
1061 | |
1062 | free_vqs: |
1063 | snet_free_vqs(snet); |
1064 | put_device: |
1065 | put_device(dev: &snet->vdpa.dev); |
1066 | free_irqs: |
1067 | if (!pf_irqs) |
1068 | pci_free_irq_vectors(dev: pdev); |
1069 | return ret; |
1070 | } |
1071 | |
1072 | static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
1073 | { |
1074 | if (pdev->is_virtfn) |
1075 | return snet_vdpa_probe_vf(pdev); |
1076 | else |
1077 | return snet_vdpa_probe_pf(pdev); |
1078 | } |
1079 | |
1080 | static void snet_vdpa_remove_pf(struct pci_dev *pdev) |
1081 | { |
1082 | struct psnet *psnet = pci_get_drvdata(pdev); |
1083 | |
1084 | pci_disable_sriov(dev: pdev); |
1085 | /* If IRQs are allocated from the PF, we should free the IRQs */ |
1086 | if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) |
1087 | pci_free_irq_vectors(dev: pdev); |
1088 | |
1089 | snet_free_cfg(cfg: &psnet->cfg); |
1090 | kfree(objp: psnet); |
1091 | } |
1092 | |
1093 | static void snet_vdpa_remove_vf(struct pci_dev *pdev) |
1094 | { |
1095 | struct snet *snet = pci_get_drvdata(pdev); |
1096 | struct psnet *psnet = snet->psnet; |
1097 | |
1098 | vdpa_unregister_device(vdev: &snet->vdpa); |
1099 | snet_free_vqs(snet); |
1100 | /* If IRQs are allocated from the VF, we should free the IRQs */ |
1101 | if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF)) |
1102 | pci_free_irq_vectors(dev: pdev); |
1103 | } |
1104 | |
1105 | static void snet_vdpa_remove(struct pci_dev *pdev) |
1106 | { |
1107 | if (pdev->is_virtfn) |
1108 | snet_vdpa_remove_vf(pdev); |
1109 | else |
1110 | snet_vdpa_remove_pf(pdev); |
1111 | } |
1112 | |
1113 | static struct pci_device_id snet_driver_pci_ids[] = { |
1114 | { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID, |
1115 | PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) }, |
1116 | { 0 }, |
1117 | }; |
1118 | |
1119 | MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids); |
1120 | |
1121 | static struct pci_driver snet_vdpa_driver = { |
1122 | .name = "snet-vdpa-driver" , |
1123 | .id_table = snet_driver_pci_ids, |
1124 | .probe = snet_vdpa_probe, |
1125 | .remove = snet_vdpa_remove, |
1126 | }; |
1127 | |
1128 | module_pci_driver(snet_vdpa_driver); |
1129 | |
1130 | MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>" ); |
1131 | MODULE_DESCRIPTION("SolidRun vDPA driver" ); |
1132 | MODULE_LICENSE("GPL v2" ); |
1133 | |