1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
2 | /* Copyright (c) 2021, Microsoft Corporation. */ |
3 | |
4 | #include <linux/module.h> |
5 | #include <linux/pci.h> |
6 | #include <linux/utsname.h> |
7 | #include <linux/version.h> |
8 | |
9 | #include <net/mana/mana.h> |
10 | |
11 | static u32 mana_gd_r32(struct gdma_context *g, u64 offset) |
12 | { |
13 | return readl(addr: g->bar0_va + offset); |
14 | } |
15 | |
16 | static u64 mana_gd_r64(struct gdma_context *g, u64 offset) |
17 | { |
18 | return readq(addr: g->bar0_va + offset); |
19 | } |
20 | |
21 | static void mana_gd_init_pf_regs(struct pci_dev *pdev) |
22 | { |
23 | struct gdma_context *gc = pci_get_drvdata(pdev); |
24 | void __iomem *sriov_base_va; |
25 | u64 sriov_base_off; |
26 | |
27 | gc->db_page_size = mana_gd_r32(g: gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF; |
28 | gc->db_page_base = gc->bar0_va + |
29 | mana_gd_r64(g: gc, GDMA_PF_REG_DB_PAGE_OFF); |
30 | |
31 | sriov_base_off = mana_gd_r64(g: gc, GDMA_SRIOV_REG_CFG_BASE_OFF); |
32 | |
33 | sriov_base_va = gc->bar0_va + sriov_base_off; |
34 | gc->shm_base = sriov_base_va + |
35 | mana_gd_r64(g: gc, offset: sriov_base_off + GDMA_PF_REG_SHM_OFF); |
36 | } |
37 | |
38 | static void mana_gd_init_vf_regs(struct pci_dev *pdev) |
39 | { |
40 | struct gdma_context *gc = pci_get_drvdata(pdev); |
41 | |
42 | gc->db_page_size = mana_gd_r32(g: gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; |
43 | |
44 | gc->db_page_base = gc->bar0_va + |
45 | mana_gd_r64(g: gc, GDMA_REG_DB_PAGE_OFFSET); |
46 | |
47 | gc->phys_db_page_base = gc->bar0_pa + |
48 | mana_gd_r64(g: gc, GDMA_REG_DB_PAGE_OFFSET); |
49 | |
50 | gc->shm_base = gc->bar0_va + mana_gd_r64(g: gc, GDMA_REG_SHM_OFFSET); |
51 | } |
52 | |
53 | static void mana_gd_init_registers(struct pci_dev *pdev) |
54 | { |
55 | struct gdma_context *gc = pci_get_drvdata(pdev); |
56 | |
57 | if (gc->is_pf) |
58 | mana_gd_init_pf_regs(pdev); |
59 | else |
60 | mana_gd_init_vf_regs(pdev); |
61 | } |
62 | |
63 | static int mana_gd_query_max_resources(struct pci_dev *pdev) |
64 | { |
65 | struct gdma_context *gc = pci_get_drvdata(pdev); |
66 | struct gdma_query_max_resources_resp resp = {}; |
67 | struct gdma_general_req req = {}; |
68 | int err; |
69 | |
70 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_QUERY_MAX_RESOURCES, |
71 | req_size: sizeof(req), resp_size: sizeof(resp)); |
72 | |
73 | err = mana_gd_send_request(gc, req_len: sizeof(req), req: &req, resp_len: sizeof(resp), resp: &resp); |
74 | if (err || resp.hdr.status) { |
75 | dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n" , |
76 | err, resp.hdr.status); |
77 | return err ? err : -EPROTO; |
78 | } |
79 | |
80 | if (gc->num_msix_usable > resp.max_msix) |
81 | gc->num_msix_usable = resp.max_msix; |
82 | |
83 | if (gc->num_msix_usable <= 1) |
84 | return -ENOSPC; |
85 | |
86 | gc->max_num_queues = num_online_cpus(); |
87 | if (gc->max_num_queues > MANA_MAX_NUM_QUEUES) |
88 | gc->max_num_queues = MANA_MAX_NUM_QUEUES; |
89 | |
90 | if (gc->max_num_queues > resp.max_eq) |
91 | gc->max_num_queues = resp.max_eq; |
92 | |
93 | if (gc->max_num_queues > resp.max_cq) |
94 | gc->max_num_queues = resp.max_cq; |
95 | |
96 | if (gc->max_num_queues > resp.max_sq) |
97 | gc->max_num_queues = resp.max_sq; |
98 | |
99 | if (gc->max_num_queues > resp.max_rq) |
100 | gc->max_num_queues = resp.max_rq; |
101 | |
102 | /* The Hardware Channel (HWC) used 1 MSI-X */ |
103 | if (gc->max_num_queues > gc->num_msix_usable - 1) |
104 | gc->max_num_queues = gc->num_msix_usable - 1; |
105 | |
106 | return 0; |
107 | } |
108 | |
109 | static int mana_gd_query_hwc_timeout(struct pci_dev *pdev, u32 *timeout_val) |
110 | { |
111 | struct gdma_context *gc = pci_get_drvdata(pdev); |
112 | struct gdma_query_hwc_timeout_resp resp = {}; |
113 | struct gdma_query_hwc_timeout_req req = {}; |
114 | int err; |
115 | |
116 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_QUERY_HWC_TIMEOUT, |
117 | req_size: sizeof(req), resp_size: sizeof(resp)); |
118 | req.timeout_ms = *timeout_val; |
119 | err = mana_gd_send_request(gc, req_len: sizeof(req), req: &req, resp_len: sizeof(resp), resp: &resp); |
120 | if (err || resp.hdr.status) |
121 | return err ? err : -EPROTO; |
122 | |
123 | *timeout_val = resp.timeout_ms; |
124 | |
125 | return 0; |
126 | } |
127 | |
128 | static int mana_gd_detect_devices(struct pci_dev *pdev) |
129 | { |
130 | struct gdma_context *gc = pci_get_drvdata(pdev); |
131 | struct gdma_list_devices_resp resp = {}; |
132 | struct gdma_general_req req = {}; |
133 | struct gdma_dev_id dev; |
134 | u32 i, max_num_devs; |
135 | u16 dev_type; |
136 | int err; |
137 | |
138 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_LIST_DEVICES, req_size: sizeof(req), |
139 | resp_size: sizeof(resp)); |
140 | |
141 | err = mana_gd_send_request(gc, req_len: sizeof(req), req: &req, resp_len: sizeof(resp), resp: &resp); |
142 | if (err || resp.hdr.status) { |
143 | dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n" , err, |
144 | resp.hdr.status); |
145 | return err ? err : -EPROTO; |
146 | } |
147 | |
148 | max_num_devs = min_t(u32, MAX_NUM_GDMA_DEVICES, resp.num_of_devs); |
149 | |
150 | for (i = 0; i < max_num_devs; i++) { |
151 | dev = resp.devs[i]; |
152 | dev_type = dev.type; |
153 | |
154 | /* HWC is already detected in mana_hwc_create_channel(). */ |
155 | if (dev_type == GDMA_DEVICE_HWC) |
156 | continue; |
157 | |
158 | if (dev_type == GDMA_DEVICE_MANA) { |
159 | gc->mana.gdma_context = gc; |
160 | gc->mana.dev_id = dev; |
161 | } else if (dev_type == GDMA_DEVICE_MANA_IB) { |
162 | gc->mana_ib.dev_id = dev; |
163 | gc->mana_ib.gdma_context = gc; |
164 | } |
165 | } |
166 | |
167 | return gc->mana.dev_id.type == 0 ? -ENODEV : 0; |
168 | } |
169 | |
170 | int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, |
171 | u32 resp_len, void *resp) |
172 | { |
173 | struct hw_channel_context *hwc = gc->hwc.driver_data; |
174 | |
175 | return mana_hwc_send_request(hwc, req_len, req, resp_len, resp); |
176 | } |
177 | EXPORT_SYMBOL_NS(mana_gd_send_request, NET_MANA); |
178 | |
179 | int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, |
180 | struct gdma_mem_info *gmi) |
181 | { |
182 | dma_addr_t dma_handle; |
183 | void *buf; |
184 | |
185 | if (length < PAGE_SIZE || !is_power_of_2(n: length)) |
186 | return -EINVAL; |
187 | |
188 | gmi->dev = gc->dev; |
189 | buf = dma_alloc_coherent(dev: gmi->dev, size: length, dma_handle: &dma_handle, GFP_KERNEL); |
190 | if (!buf) |
191 | return -ENOMEM; |
192 | |
193 | gmi->dma_handle = dma_handle; |
194 | gmi->virt_addr = buf; |
195 | gmi->length = length; |
196 | |
197 | return 0; |
198 | } |
199 | |
200 | void mana_gd_free_memory(struct gdma_mem_info *gmi) |
201 | { |
202 | dma_free_coherent(dev: gmi->dev, size: gmi->length, cpu_addr: gmi->virt_addr, |
203 | dma_handle: gmi->dma_handle); |
204 | } |
205 | |
206 | static int mana_gd_create_hw_eq(struct gdma_context *gc, |
207 | struct gdma_queue *queue) |
208 | { |
209 | struct gdma_create_queue_resp resp = {}; |
210 | struct gdma_create_queue_req req = {}; |
211 | int err; |
212 | |
213 | if (queue->type != GDMA_EQ) |
214 | return -EINVAL; |
215 | |
216 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_CREATE_QUEUE, |
217 | req_size: sizeof(req), resp_size: sizeof(resp)); |
218 | |
219 | req.hdr.dev_id = queue->gdma_dev->dev_id; |
220 | req.type = queue->type; |
221 | req.pdid = queue->gdma_dev->pdid; |
222 | req.doolbell_id = queue->gdma_dev->doorbell; |
223 | req.gdma_region = queue->mem_info.dma_region_handle; |
224 | req.queue_size = queue->queue_size; |
225 | req.log2_throttle_limit = queue->eq.log2_throttle_limit; |
226 | req.eq_pci_msix_index = queue->eq.msix_index; |
227 | |
228 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
229 | if (err || resp.hdr.status) { |
230 | dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n" , err, |
231 | resp.hdr.status); |
232 | return err ? err : -EPROTO; |
233 | } |
234 | |
235 | queue->id = resp.queue_index; |
236 | queue->eq.disable_needed = true; |
237 | queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; |
238 | return 0; |
239 | } |
240 | |
241 | static int mana_gd_disable_queue(struct gdma_queue *queue) |
242 | { |
243 | struct gdma_context *gc = queue->gdma_dev->gdma_context; |
244 | struct gdma_disable_queue_req req = {}; |
245 | struct gdma_general_resp resp = {}; |
246 | int err; |
247 | |
248 | WARN_ON(queue->type != GDMA_EQ); |
249 | |
250 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_DISABLE_QUEUE, |
251 | req_size: sizeof(req), resp_size: sizeof(resp)); |
252 | |
253 | req.hdr.dev_id = queue->gdma_dev->dev_id; |
254 | req.type = queue->type; |
255 | req.queue_index = queue->id; |
256 | req.alloc_res_id_on_creation = 1; |
257 | |
258 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
259 | if (err || resp.hdr.status) { |
260 | dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n" , err, |
261 | resp.hdr.status); |
262 | return err ? err : -EPROTO; |
263 | } |
264 | |
265 | return 0; |
266 | } |
267 | |
268 | #define DOORBELL_OFFSET_SQ 0x0 |
269 | #define DOORBELL_OFFSET_RQ 0x400 |
270 | #define DOORBELL_OFFSET_CQ 0x800 |
271 | #define DOORBELL_OFFSET_EQ 0xFF8 |
272 | |
273 | static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, |
274 | enum gdma_queue_type q_type, u32 qid, |
275 | u32 tail_ptr, u8 num_req) |
276 | { |
277 | void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; |
278 | union gdma_doorbell_entry e = {}; |
279 | |
280 | switch (q_type) { |
281 | case GDMA_EQ: |
282 | e.eq.id = qid; |
283 | e.eq.tail_ptr = tail_ptr; |
284 | e.eq.arm = num_req; |
285 | |
286 | addr += DOORBELL_OFFSET_EQ; |
287 | break; |
288 | |
289 | case GDMA_CQ: |
290 | e.cq.id = qid; |
291 | e.cq.tail_ptr = tail_ptr; |
292 | e.cq.arm = num_req; |
293 | |
294 | addr += DOORBELL_OFFSET_CQ; |
295 | break; |
296 | |
297 | case GDMA_RQ: |
298 | e.rq.id = qid; |
299 | e.rq.tail_ptr = tail_ptr; |
300 | e.rq.wqe_cnt = num_req; |
301 | |
302 | addr += DOORBELL_OFFSET_RQ; |
303 | break; |
304 | |
305 | case GDMA_SQ: |
306 | e.sq.id = qid; |
307 | e.sq.tail_ptr = tail_ptr; |
308 | |
309 | addr += DOORBELL_OFFSET_SQ; |
310 | break; |
311 | |
312 | default: |
313 | WARN_ON(1); |
314 | return; |
315 | } |
316 | |
317 | /* Ensure all writes are done before ring doorbell */ |
318 | wmb(); |
319 | |
320 | writeq(val: e.as_uint64, addr); |
321 | } |
322 | |
323 | void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) |
324 | { |
325 | /* Hardware Spec specifies that software client should set 0 for |
326 | * wqe_cnt for Receive Queues. This value is not used in Send Queues. |
327 | */ |
328 | mana_gd_ring_doorbell(gc, db_index: queue->gdma_dev->doorbell, q_type: queue->type, |
329 | qid: queue->id, tail_ptr: queue->head * GDMA_WQE_BU_SIZE, num_req: 0); |
330 | } |
331 | |
332 | void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) |
333 | { |
334 | struct gdma_context *gc = cq->gdma_dev->gdma_context; |
335 | |
336 | u32 num_cqe = cq->queue_size / GDMA_CQE_SIZE; |
337 | |
338 | u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); |
339 | |
340 | mana_gd_ring_doorbell(gc, db_index: cq->gdma_dev->doorbell, q_type: cq->type, qid: cq->id, |
341 | tail_ptr: head, num_req: arm_bit); |
342 | } |
343 | |
344 | static void mana_gd_process_eqe(struct gdma_queue *eq) |
345 | { |
346 | u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); |
347 | struct gdma_context *gc = eq->gdma_dev->gdma_context; |
348 | struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr; |
349 | union gdma_eqe_info eqe_info; |
350 | enum gdma_eqe_type type; |
351 | struct gdma_event event; |
352 | struct gdma_queue *cq; |
353 | struct gdma_eqe *eqe; |
354 | u32 cq_id; |
355 | |
356 | eqe = &eq_eqe_ptr[head]; |
357 | eqe_info.as_uint32 = eqe->eqe_info; |
358 | type = eqe_info.type; |
359 | |
360 | switch (type) { |
361 | case GDMA_EQE_COMPLETION: |
362 | cq_id = eqe->details[0] & 0xFFFFFF; |
363 | if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs)) |
364 | break; |
365 | |
366 | cq = gc->cq_table[cq_id]; |
367 | if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id)) |
368 | break; |
369 | |
370 | if (cq->cq.callback) |
371 | cq->cq.callback(cq->cq.context, cq); |
372 | |
373 | break; |
374 | |
375 | case GDMA_EQE_TEST_EVENT: |
376 | gc->test_event_eq_id = eq->id; |
377 | complete(&gc->eq_test_event); |
378 | break; |
379 | |
380 | case GDMA_EQE_HWC_INIT_EQ_ID_DB: |
381 | case GDMA_EQE_HWC_INIT_DATA: |
382 | case GDMA_EQE_HWC_INIT_DONE: |
383 | if (!eq->eq.callback) |
384 | break; |
385 | |
386 | event.type = type; |
387 | memcpy(&event.details, &eqe->details, GDMA_EVENT_DATA_SIZE); |
388 | eq->eq.callback(eq->eq.context, eq, &event); |
389 | break; |
390 | |
391 | default: |
392 | break; |
393 | } |
394 | } |
395 | |
396 | static void mana_gd_process_eq_events(void *arg) |
397 | { |
398 | u32 owner_bits, new_bits, old_bits; |
399 | union gdma_eqe_info eqe_info; |
400 | struct gdma_eqe *eq_eqe_ptr; |
401 | struct gdma_queue *eq = arg; |
402 | struct gdma_context *gc; |
403 | struct gdma_eqe *eqe; |
404 | u32 head, num_eqe; |
405 | int i; |
406 | |
407 | gc = eq->gdma_dev->gdma_context; |
408 | |
409 | num_eqe = eq->queue_size / GDMA_EQE_SIZE; |
410 | eq_eqe_ptr = eq->queue_mem_ptr; |
411 | |
412 | /* Process up to 5 EQEs at a time, and update the HW head. */ |
413 | for (i = 0; i < 5; i++) { |
414 | eqe = &eq_eqe_ptr[eq->head % num_eqe]; |
415 | eqe_info.as_uint32 = eqe->eqe_info; |
416 | owner_bits = eqe_info.owner_bits; |
417 | |
418 | old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK; |
419 | /* No more entries */ |
420 | if (owner_bits == old_bits) { |
421 | /* return here without ringing the doorbell */ |
422 | if (i == 0) |
423 | return; |
424 | break; |
425 | } |
426 | |
427 | new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK; |
428 | if (owner_bits != new_bits) { |
429 | dev_err(gc->dev, "EQ %d: overflow detected\n" , eq->id); |
430 | break; |
431 | } |
432 | |
433 | /* Per GDMA spec, rmb is necessary after checking owner_bits, before |
434 | * reading eqe. |
435 | */ |
436 | rmb(); |
437 | |
438 | mana_gd_process_eqe(eq); |
439 | |
440 | eq->head++; |
441 | } |
442 | |
443 | head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); |
444 | |
445 | mana_gd_ring_doorbell(gc, db_index: eq->gdma_dev->doorbell, q_type: eq->type, qid: eq->id, |
446 | tail_ptr: head, SET_ARM_BIT); |
447 | } |
448 | |
449 | static int mana_gd_register_irq(struct gdma_queue *queue, |
450 | const struct gdma_queue_spec *spec) |
451 | { |
452 | struct gdma_dev *gd = queue->gdma_dev; |
453 | struct gdma_irq_context *gic; |
454 | struct gdma_context *gc; |
455 | unsigned int msi_index; |
456 | unsigned long flags; |
457 | struct device *dev; |
458 | int err = 0; |
459 | |
460 | gc = gd->gdma_context; |
461 | dev = gc->dev; |
462 | msi_index = spec->eq.msix_index; |
463 | |
464 | if (msi_index >= gc->num_msix_usable) { |
465 | err = -ENOSPC; |
466 | dev_err(dev, "Register IRQ err:%d, msi:%u nMSI:%u" , |
467 | err, msi_index, gc->num_msix_usable); |
468 | |
469 | return err; |
470 | } |
471 | |
472 | queue->eq.msix_index = msi_index; |
473 | gic = &gc->irq_contexts[msi_index]; |
474 | |
475 | spin_lock_irqsave(&gic->lock, flags); |
476 | list_add_rcu(new: &queue->entry, head: &gic->eq_list); |
477 | spin_unlock_irqrestore(lock: &gic->lock, flags); |
478 | |
479 | return 0; |
480 | } |
481 | |
482 | static void mana_gd_deregiser_irq(struct gdma_queue *queue) |
483 | { |
484 | struct gdma_dev *gd = queue->gdma_dev; |
485 | struct gdma_irq_context *gic; |
486 | struct gdma_context *gc; |
487 | unsigned int msix_index; |
488 | unsigned long flags; |
489 | struct gdma_queue *eq; |
490 | |
491 | gc = gd->gdma_context; |
492 | |
493 | /* At most num_online_cpus() + 1 interrupts are used. */ |
494 | msix_index = queue->eq.msix_index; |
495 | if (WARN_ON(msix_index >= gc->num_msix_usable)) |
496 | return; |
497 | |
498 | gic = &gc->irq_contexts[msix_index]; |
499 | spin_lock_irqsave(&gic->lock, flags); |
500 | list_for_each_entry_rcu(eq, &gic->eq_list, entry) { |
501 | if (queue == eq) { |
502 | list_del_rcu(entry: &eq->entry); |
503 | break; |
504 | } |
505 | } |
506 | spin_unlock_irqrestore(lock: &gic->lock, flags); |
507 | |
508 | queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; |
509 | synchronize_rcu(); |
510 | } |
511 | |
512 | int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq) |
513 | { |
514 | struct gdma_generate_test_event_req req = {}; |
515 | struct gdma_general_resp resp = {}; |
516 | struct device *dev = gc->dev; |
517 | int err; |
518 | |
519 | mutex_lock(&gc->eq_test_event_mutex); |
520 | |
521 | init_completion(x: &gc->eq_test_event); |
522 | gc->test_event_eq_id = INVALID_QUEUE_ID; |
523 | |
524 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_GENERATE_TEST_EQE, |
525 | req_size: sizeof(req), resp_size: sizeof(resp)); |
526 | |
527 | req.hdr.dev_id = eq->gdma_dev->dev_id; |
528 | req.queue_index = eq->id; |
529 | |
530 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
531 | if (err) { |
532 | dev_err(dev, "test_eq failed: %d\n" , err); |
533 | goto out; |
534 | } |
535 | |
536 | err = -EPROTO; |
537 | |
538 | if (resp.hdr.status) { |
539 | dev_err(dev, "test_eq failed: 0x%x\n" , resp.hdr.status); |
540 | goto out; |
541 | } |
542 | |
543 | if (!wait_for_completion_timeout(x: &gc->eq_test_event, timeout: 30 * HZ)) { |
544 | dev_err(dev, "test_eq timed out on queue %d\n" , eq->id); |
545 | goto out; |
546 | } |
547 | |
548 | if (eq->id != gc->test_event_eq_id) { |
549 | dev_err(dev, "test_eq got an event on wrong queue %d (%d)\n" , |
550 | gc->test_event_eq_id, eq->id); |
551 | goto out; |
552 | } |
553 | |
554 | err = 0; |
555 | out: |
556 | mutex_unlock(lock: &gc->eq_test_event_mutex); |
557 | return err; |
558 | } |
559 | |
560 | static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets, |
561 | struct gdma_queue *queue) |
562 | { |
563 | int err; |
564 | |
565 | if (flush_evenets) { |
566 | err = mana_gd_test_eq(gc, eq: queue); |
567 | if (err) |
568 | dev_warn(gc->dev, "Failed to flush EQ: %d\n" , err); |
569 | } |
570 | |
571 | mana_gd_deregiser_irq(queue); |
572 | |
573 | if (queue->eq.disable_needed) |
574 | mana_gd_disable_queue(queue); |
575 | } |
576 | |
577 | static int mana_gd_create_eq(struct gdma_dev *gd, |
578 | const struct gdma_queue_spec *spec, |
579 | bool create_hwq, struct gdma_queue *queue) |
580 | { |
581 | struct gdma_context *gc = gd->gdma_context; |
582 | struct device *dev = gc->dev; |
583 | u32 log2_num_entries; |
584 | int err; |
585 | |
586 | queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; |
587 | queue->id = INVALID_QUEUE_ID; |
588 | |
589 | log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE); |
590 | |
591 | if (spec->eq.log2_throttle_limit > log2_num_entries) { |
592 | dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n" , |
593 | spec->eq.log2_throttle_limit, log2_num_entries); |
594 | return -EINVAL; |
595 | } |
596 | |
597 | err = mana_gd_register_irq(queue, spec); |
598 | if (err) { |
599 | dev_err(dev, "Failed to register irq: %d\n" , err); |
600 | return err; |
601 | } |
602 | |
603 | queue->eq.callback = spec->eq.callback; |
604 | queue->eq.context = spec->eq.context; |
605 | queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); |
606 | queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1; |
607 | |
608 | if (create_hwq) { |
609 | err = mana_gd_create_hw_eq(gc, queue); |
610 | if (err) |
611 | goto out; |
612 | |
613 | err = mana_gd_test_eq(gc, eq: queue); |
614 | if (err) |
615 | goto out; |
616 | } |
617 | |
618 | return 0; |
619 | out: |
620 | dev_err(dev, "Failed to create EQ: %d\n" , err); |
621 | mana_gd_destroy_eq(gc, flush_evenets: false, queue); |
622 | return err; |
623 | } |
624 | |
625 | static void mana_gd_create_cq(const struct gdma_queue_spec *spec, |
626 | struct gdma_queue *queue) |
627 | { |
628 | u32 log2_num_entries = ilog2(spec->queue_size / GDMA_CQE_SIZE); |
629 | |
630 | queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); |
631 | queue->cq.parent = spec->cq.parent_eq; |
632 | queue->cq.context = spec->cq.context; |
633 | queue->cq.callback = spec->cq.callback; |
634 | } |
635 | |
636 | static void mana_gd_destroy_cq(struct gdma_context *gc, |
637 | struct gdma_queue *queue) |
638 | { |
639 | u32 id = queue->id; |
640 | |
641 | if (id >= gc->max_num_cqs) |
642 | return; |
643 | |
644 | if (!gc->cq_table[id]) |
645 | return; |
646 | |
647 | gc->cq_table[id] = NULL; |
648 | } |
649 | |
650 | int mana_gd_create_hwc_queue(struct gdma_dev *gd, |
651 | const struct gdma_queue_spec *spec, |
652 | struct gdma_queue **queue_ptr) |
653 | { |
654 | struct gdma_context *gc = gd->gdma_context; |
655 | struct gdma_mem_info *gmi; |
656 | struct gdma_queue *queue; |
657 | int err; |
658 | |
659 | queue = kzalloc(size: sizeof(*queue), GFP_KERNEL); |
660 | if (!queue) |
661 | return -ENOMEM; |
662 | |
663 | gmi = &queue->mem_info; |
664 | err = mana_gd_alloc_memory(gc, length: spec->queue_size, gmi); |
665 | if (err) |
666 | goto free_q; |
667 | |
668 | queue->head = 0; |
669 | queue->tail = 0; |
670 | queue->queue_mem_ptr = gmi->virt_addr; |
671 | queue->queue_size = spec->queue_size; |
672 | queue->monitor_avl_buf = spec->monitor_avl_buf; |
673 | queue->type = spec->type; |
674 | queue->gdma_dev = gd; |
675 | |
676 | if (spec->type == GDMA_EQ) |
677 | err = mana_gd_create_eq(gd, spec, create_hwq: false, queue); |
678 | else if (spec->type == GDMA_CQ) |
679 | mana_gd_create_cq(spec, queue); |
680 | |
681 | if (err) |
682 | goto out; |
683 | |
684 | *queue_ptr = queue; |
685 | return 0; |
686 | out: |
687 | mana_gd_free_memory(gmi); |
688 | free_q: |
689 | kfree(objp: queue); |
690 | return err; |
691 | } |
692 | |
693 | int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle) |
694 | { |
695 | struct gdma_destroy_dma_region_req req = {}; |
696 | struct gdma_general_resp resp = {}; |
697 | int err; |
698 | |
699 | if (dma_region_handle == GDMA_INVALID_DMA_REGION) |
700 | return 0; |
701 | |
702 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_DESTROY_DMA_REGION, req_size: sizeof(req), |
703 | resp_size: sizeof(resp)); |
704 | req.dma_region_handle = dma_region_handle; |
705 | |
706 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
707 | if (err || resp.hdr.status) { |
708 | dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n" , |
709 | err, resp.hdr.status); |
710 | return -EPROTO; |
711 | } |
712 | |
713 | return 0; |
714 | } |
715 | EXPORT_SYMBOL_NS(mana_gd_destroy_dma_region, NET_MANA); |
716 | |
717 | static int mana_gd_create_dma_region(struct gdma_dev *gd, |
718 | struct gdma_mem_info *gmi) |
719 | { |
720 | unsigned int num_page = gmi->length / PAGE_SIZE; |
721 | struct gdma_create_dma_region_req *req = NULL; |
722 | struct gdma_create_dma_region_resp resp = {}; |
723 | struct gdma_context *gc = gd->gdma_context; |
724 | struct hw_channel_context *hwc; |
725 | u32 length = gmi->length; |
726 | size_t req_msg_size; |
727 | int err; |
728 | int i; |
729 | |
730 | if (length < PAGE_SIZE || !is_power_of_2(n: length)) |
731 | return -EINVAL; |
732 | |
733 | if (offset_in_page(gmi->virt_addr) != 0) |
734 | return -EINVAL; |
735 | |
736 | hwc = gc->hwc.driver_data; |
737 | req_msg_size = struct_size(req, page_addr_list, num_page); |
738 | if (req_msg_size > hwc->max_req_msg_size) |
739 | return -EINVAL; |
740 | |
741 | req = kzalloc(size: req_msg_size, GFP_KERNEL); |
742 | if (!req) |
743 | return -ENOMEM; |
744 | |
745 | mana_gd_init_req_hdr(hdr: &req->hdr, code: GDMA_CREATE_DMA_REGION, |
746 | req_size: req_msg_size, resp_size: sizeof(resp)); |
747 | req->length = length; |
748 | req->offset_in_page = 0; |
749 | req->gdma_page_type = GDMA_PAGE_TYPE_4K; |
750 | req->page_count = num_page; |
751 | req->page_addr_list_len = num_page; |
752 | |
753 | for (i = 0; i < num_page; i++) |
754 | req->page_addr_list[i] = gmi->dma_handle + i * PAGE_SIZE; |
755 | |
756 | err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp); |
757 | if (err) |
758 | goto out; |
759 | |
760 | if (resp.hdr.status || |
761 | resp.dma_region_handle == GDMA_INVALID_DMA_REGION) { |
762 | dev_err(gc->dev, "Failed to create DMA region: 0x%x\n" , |
763 | resp.hdr.status); |
764 | err = -EPROTO; |
765 | goto out; |
766 | } |
767 | |
768 | gmi->dma_region_handle = resp.dma_region_handle; |
769 | out: |
770 | kfree(objp: req); |
771 | return err; |
772 | } |
773 | |
774 | int mana_gd_create_mana_eq(struct gdma_dev *gd, |
775 | const struct gdma_queue_spec *spec, |
776 | struct gdma_queue **queue_ptr) |
777 | { |
778 | struct gdma_context *gc = gd->gdma_context; |
779 | struct gdma_mem_info *gmi; |
780 | struct gdma_queue *queue; |
781 | int err; |
782 | |
783 | if (spec->type != GDMA_EQ) |
784 | return -EINVAL; |
785 | |
786 | queue = kzalloc(size: sizeof(*queue), GFP_KERNEL); |
787 | if (!queue) |
788 | return -ENOMEM; |
789 | |
790 | gmi = &queue->mem_info; |
791 | err = mana_gd_alloc_memory(gc, length: spec->queue_size, gmi); |
792 | if (err) |
793 | goto free_q; |
794 | |
795 | err = mana_gd_create_dma_region(gd, gmi); |
796 | if (err) |
797 | goto out; |
798 | |
799 | queue->head = 0; |
800 | queue->tail = 0; |
801 | queue->queue_mem_ptr = gmi->virt_addr; |
802 | queue->queue_size = spec->queue_size; |
803 | queue->monitor_avl_buf = spec->monitor_avl_buf; |
804 | queue->type = spec->type; |
805 | queue->gdma_dev = gd; |
806 | |
807 | err = mana_gd_create_eq(gd, spec, create_hwq: true, queue); |
808 | if (err) |
809 | goto out; |
810 | |
811 | *queue_ptr = queue; |
812 | return 0; |
813 | out: |
814 | mana_gd_free_memory(gmi); |
815 | free_q: |
816 | kfree(objp: queue); |
817 | return err; |
818 | } |
819 | EXPORT_SYMBOL_NS(mana_gd_create_mana_eq, NET_MANA); |
820 | |
821 | int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, |
822 | const struct gdma_queue_spec *spec, |
823 | struct gdma_queue **queue_ptr) |
824 | { |
825 | struct gdma_context *gc = gd->gdma_context; |
826 | struct gdma_mem_info *gmi; |
827 | struct gdma_queue *queue; |
828 | int err; |
829 | |
830 | if (spec->type != GDMA_CQ && spec->type != GDMA_SQ && |
831 | spec->type != GDMA_RQ) |
832 | return -EINVAL; |
833 | |
834 | queue = kzalloc(size: sizeof(*queue), GFP_KERNEL); |
835 | if (!queue) |
836 | return -ENOMEM; |
837 | |
838 | gmi = &queue->mem_info; |
839 | err = mana_gd_alloc_memory(gc, length: spec->queue_size, gmi); |
840 | if (err) |
841 | goto free_q; |
842 | |
843 | err = mana_gd_create_dma_region(gd, gmi); |
844 | if (err) |
845 | goto out; |
846 | |
847 | queue->head = 0; |
848 | queue->tail = 0; |
849 | queue->queue_mem_ptr = gmi->virt_addr; |
850 | queue->queue_size = spec->queue_size; |
851 | queue->monitor_avl_buf = spec->monitor_avl_buf; |
852 | queue->type = spec->type; |
853 | queue->gdma_dev = gd; |
854 | |
855 | if (spec->type == GDMA_CQ) |
856 | mana_gd_create_cq(spec, queue); |
857 | |
858 | *queue_ptr = queue; |
859 | return 0; |
860 | out: |
861 | mana_gd_free_memory(gmi); |
862 | free_q: |
863 | kfree(objp: queue); |
864 | return err; |
865 | } |
866 | |
867 | void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) |
868 | { |
869 | struct gdma_mem_info *gmi = &queue->mem_info; |
870 | |
871 | switch (queue->type) { |
872 | case GDMA_EQ: |
873 | mana_gd_destroy_eq(gc, flush_evenets: queue->eq.disable_needed, queue); |
874 | break; |
875 | |
876 | case GDMA_CQ: |
877 | mana_gd_destroy_cq(gc, queue); |
878 | break; |
879 | |
880 | case GDMA_RQ: |
881 | break; |
882 | |
883 | case GDMA_SQ: |
884 | break; |
885 | |
886 | default: |
887 | dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n" , |
888 | queue->type); |
889 | return; |
890 | } |
891 | |
892 | mana_gd_destroy_dma_region(gc, gmi->dma_region_handle); |
893 | mana_gd_free_memory(gmi); |
894 | kfree(objp: queue); |
895 | } |
896 | EXPORT_SYMBOL_NS(mana_gd_destroy_queue, NET_MANA); |
897 | |
898 | int mana_gd_verify_vf_version(struct pci_dev *pdev) |
899 | { |
900 | struct gdma_context *gc = pci_get_drvdata(pdev); |
901 | struct gdma_verify_ver_resp resp = {}; |
902 | struct gdma_verify_ver_req req = {}; |
903 | struct hw_channel_context *hwc; |
904 | int err; |
905 | |
906 | hwc = gc->hwc.driver_data; |
907 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_VERIFY_VF_DRIVER_VERSION, |
908 | req_size: sizeof(req), resp_size: sizeof(resp)); |
909 | |
910 | req.protocol_ver_min = GDMA_PROTOCOL_FIRST; |
911 | req.protocol_ver_max = GDMA_PROTOCOL_LAST; |
912 | |
913 | req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1; |
914 | req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2; |
915 | req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3; |
916 | req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4; |
917 | |
918 | req.drv_ver = 0; /* Unused*/ |
919 | req.os_type = 0x10; /* Linux */ |
920 | req.os_ver_major = LINUX_VERSION_MAJOR; |
921 | req.os_ver_minor = LINUX_VERSION_PATCHLEVEL; |
922 | req.os_ver_build = LINUX_VERSION_SUBLEVEL; |
923 | strscpy(req.os_ver_str1, utsname()->sysname, sizeof(req.os_ver_str1)); |
924 | strscpy(req.os_ver_str2, utsname()->release, sizeof(req.os_ver_str2)); |
925 | strscpy(req.os_ver_str3, utsname()->version, sizeof(req.os_ver_str3)); |
926 | |
927 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
928 | if (err || resp.hdr.status) { |
929 | dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n" , |
930 | err, resp.hdr.status); |
931 | return err ? err : -EPROTO; |
932 | } |
933 | if (resp.pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG) { |
934 | err = mana_gd_query_hwc_timeout(pdev, timeout_val: &hwc->hwc_timeout); |
935 | if (err) { |
936 | dev_err(gc->dev, "Failed to set the hwc timeout %d\n" , err); |
937 | return err; |
938 | } |
939 | dev_dbg(gc->dev, "set the hwc timeout to %u\n" , hwc->hwc_timeout); |
940 | } |
941 | return 0; |
942 | } |
943 | |
944 | int mana_gd_register_device(struct gdma_dev *gd) |
945 | { |
946 | struct gdma_context *gc = gd->gdma_context; |
947 | struct gdma_register_device_resp resp = {}; |
948 | struct gdma_general_req req = {}; |
949 | int err; |
950 | |
951 | gd->pdid = INVALID_PDID; |
952 | gd->doorbell = INVALID_DOORBELL; |
953 | gd->gpa_mkey = INVALID_MEM_KEY; |
954 | |
955 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_REGISTER_DEVICE, req_size: sizeof(req), |
956 | resp_size: sizeof(resp)); |
957 | |
958 | req.hdr.dev_id = gd->dev_id; |
959 | |
960 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
961 | if (err || resp.hdr.status) { |
962 | dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n" , |
963 | err, resp.hdr.status); |
964 | return err ? err : -EPROTO; |
965 | } |
966 | |
967 | gd->pdid = resp.pdid; |
968 | gd->gpa_mkey = resp.gpa_mkey; |
969 | gd->doorbell = resp.db_id; |
970 | |
971 | return 0; |
972 | } |
973 | EXPORT_SYMBOL_NS(mana_gd_register_device, NET_MANA); |
974 | |
975 | int mana_gd_deregister_device(struct gdma_dev *gd) |
976 | { |
977 | struct gdma_context *gc = gd->gdma_context; |
978 | struct gdma_general_resp resp = {}; |
979 | struct gdma_general_req req = {}; |
980 | int err; |
981 | |
982 | if (gd->pdid == INVALID_PDID) |
983 | return -EINVAL; |
984 | |
985 | mana_gd_init_req_hdr(hdr: &req.hdr, code: GDMA_DEREGISTER_DEVICE, req_size: sizeof(req), |
986 | resp_size: sizeof(resp)); |
987 | |
988 | req.hdr.dev_id = gd->dev_id; |
989 | |
990 | err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); |
991 | if (err || resp.hdr.status) { |
992 | dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n" , |
993 | err, resp.hdr.status); |
994 | if (!err) |
995 | err = -EPROTO; |
996 | } |
997 | |
998 | gd->pdid = INVALID_PDID; |
999 | gd->doorbell = INVALID_DOORBELL; |
1000 | gd->gpa_mkey = INVALID_MEM_KEY; |
1001 | |
1002 | return err; |
1003 | } |
1004 | EXPORT_SYMBOL_NS(mana_gd_deregister_device, NET_MANA); |
1005 | |
1006 | u32 mana_gd_wq_avail_space(struct gdma_queue *wq) |
1007 | { |
1008 | u32 used_space = (wq->head - wq->tail) * GDMA_WQE_BU_SIZE; |
1009 | u32 wq_size = wq->queue_size; |
1010 | |
1011 | WARN_ON_ONCE(used_space > wq_size); |
1012 | |
1013 | return wq_size - used_space; |
1014 | } |
1015 | |
1016 | u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset) |
1017 | { |
1018 | u32 offset = (wqe_offset * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); |
1019 | |
1020 | WARN_ON_ONCE((offset + GDMA_WQE_BU_SIZE) > wq->queue_size); |
1021 | |
1022 | return wq->queue_mem_ptr + offset; |
1023 | } |
1024 | |
1025 | static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req, |
1026 | enum gdma_queue_type q_type, |
1027 | u32 client_oob_size, u32 sgl_data_size, |
1028 | u8 *wqe_ptr) |
1029 | { |
1030 | bool oob_in_sgl = !!(wqe_req->flags & GDMA_WR_OOB_IN_SGL); |
1031 | bool pad_data = !!(wqe_req->flags & GDMA_WR_PAD_BY_SGE0); |
1032 | struct gdma_wqe * = (struct gdma_wqe *)wqe_ptr; |
1033 | u8 *ptr; |
1034 | |
1035 | memset(header, 0, sizeof(struct gdma_wqe)); |
1036 | header->num_sge = wqe_req->num_sge; |
1037 | header->inline_oob_size_div4 = client_oob_size / sizeof(u32); |
1038 | |
1039 | if (oob_in_sgl) { |
1040 | WARN_ON_ONCE(!pad_data || wqe_req->num_sge < 2); |
1041 | |
1042 | header->client_oob_in_sgl = 1; |
1043 | |
1044 | if (pad_data) |
1045 | header->last_vbytes = wqe_req->sgl[0].size; |
1046 | } |
1047 | |
1048 | if (q_type == GDMA_SQ) |
1049 | header->client_data_unit = wqe_req->client_data_unit; |
1050 | |
1051 | /* The size of gdma_wqe + client_oob_size must be less than or equal |
1052 | * to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond |
1053 | * the queue memory buffer boundary. |
1054 | */ |
1055 | ptr = wqe_ptr + sizeof(header); |
1056 | |
1057 | if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) { |
1058 | memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size); |
1059 | |
1060 | if (client_oob_size > wqe_req->inline_oob_size) |
1061 | memset(ptr + wqe_req->inline_oob_size, 0, |
1062 | client_oob_size - wqe_req->inline_oob_size); |
1063 | } |
1064 | |
1065 | return sizeof(header) + client_oob_size; |
1066 | } |
1067 | |
1068 | static void mana_gd_write_sgl(struct gdma_queue *wq, u8 *wqe_ptr, |
1069 | const struct gdma_wqe_request *wqe_req) |
1070 | { |
1071 | u32 sgl_size = sizeof(struct gdma_sge) * wqe_req->num_sge; |
1072 | const u8 *address = (u8 *)wqe_req->sgl; |
1073 | u8 *base_ptr, *end_ptr; |
1074 | u32 size_to_end; |
1075 | |
1076 | base_ptr = wq->queue_mem_ptr; |
1077 | end_ptr = base_ptr + wq->queue_size; |
1078 | size_to_end = (u32)(end_ptr - wqe_ptr); |
1079 | |
1080 | if (size_to_end < sgl_size) { |
1081 | memcpy(wqe_ptr, address, size_to_end); |
1082 | |
1083 | wqe_ptr = base_ptr; |
1084 | address += size_to_end; |
1085 | sgl_size -= size_to_end; |
1086 | } |
1087 | |
1088 | memcpy(wqe_ptr, address, sgl_size); |
1089 | } |
1090 | |
1091 | int mana_gd_post_work_request(struct gdma_queue *wq, |
1092 | const struct gdma_wqe_request *wqe_req, |
1093 | struct gdma_posted_wqe_info *wqe_info) |
1094 | { |
1095 | u32 client_oob_size = wqe_req->inline_oob_size; |
1096 | struct gdma_context *gc; |
1097 | u32 sgl_data_size; |
1098 | u32 max_wqe_size; |
1099 | u32 wqe_size; |
1100 | u8 *wqe_ptr; |
1101 | |
1102 | if (wqe_req->num_sge == 0) |
1103 | return -EINVAL; |
1104 | |
1105 | if (wq->type == GDMA_RQ) { |
1106 | if (client_oob_size != 0) |
1107 | return -EINVAL; |
1108 | |
1109 | client_oob_size = INLINE_OOB_SMALL_SIZE; |
1110 | |
1111 | max_wqe_size = GDMA_MAX_RQE_SIZE; |
1112 | } else { |
1113 | if (client_oob_size != INLINE_OOB_SMALL_SIZE && |
1114 | client_oob_size != INLINE_OOB_LARGE_SIZE) |
1115 | return -EINVAL; |
1116 | |
1117 | max_wqe_size = GDMA_MAX_SQE_SIZE; |
1118 | } |
1119 | |
1120 | sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge; |
1121 | wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size + |
1122 | sgl_data_size, GDMA_WQE_BU_SIZE); |
1123 | if (wqe_size > max_wqe_size) |
1124 | return -EINVAL; |
1125 | |
1126 | if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) { |
1127 | gc = wq->gdma_dev->gdma_context; |
1128 | dev_err(gc->dev, "unsuccessful flow control!\n" ); |
1129 | return -ENOSPC; |
1130 | } |
1131 | |
1132 | if (wqe_info) |
1133 | wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; |
1134 | |
1135 | wqe_ptr = mana_gd_get_wqe_ptr(wq, wqe_offset: wq->head); |
1136 | wqe_ptr += mana_gd_write_client_oob(wqe_req, q_type: wq->type, client_oob_size, |
1137 | sgl_data_size, wqe_ptr); |
1138 | if (wqe_ptr >= (u8 *)wq->queue_mem_ptr + wq->queue_size) |
1139 | wqe_ptr -= wq->queue_size; |
1140 | |
1141 | mana_gd_write_sgl(wq, wqe_ptr, wqe_req); |
1142 | |
1143 | wq->head += wqe_size / GDMA_WQE_BU_SIZE; |
1144 | |
1145 | return 0; |
1146 | } |
1147 | |
1148 | int mana_gd_post_and_ring(struct gdma_queue *queue, |
1149 | const struct gdma_wqe_request *wqe_req, |
1150 | struct gdma_posted_wqe_info *wqe_info) |
1151 | { |
1152 | struct gdma_context *gc = queue->gdma_dev->gdma_context; |
1153 | int err; |
1154 | |
1155 | err = mana_gd_post_work_request(wq: queue, wqe_req, wqe_info); |
1156 | if (err) |
1157 | return err; |
1158 | |
1159 | mana_gd_wq_ring_doorbell(gc, queue); |
1160 | |
1161 | return 0; |
1162 | } |
1163 | |
1164 | static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp) |
1165 | { |
1166 | unsigned int num_cqe = cq->queue_size / sizeof(struct gdma_cqe); |
1167 | struct gdma_cqe *cq_cqe = cq->queue_mem_ptr; |
1168 | u32 owner_bits, new_bits, old_bits; |
1169 | struct gdma_cqe *cqe; |
1170 | |
1171 | cqe = &cq_cqe[cq->head % num_cqe]; |
1172 | owner_bits = cqe->cqe_info.owner_bits; |
1173 | |
1174 | old_bits = (cq->head / num_cqe - 1) & GDMA_CQE_OWNER_MASK; |
1175 | /* Return 0 if no more entries. */ |
1176 | if (owner_bits == old_bits) |
1177 | return 0; |
1178 | |
1179 | new_bits = (cq->head / num_cqe) & GDMA_CQE_OWNER_MASK; |
1180 | /* Return -1 if overflow detected. */ |
1181 | if (WARN_ON_ONCE(owner_bits != new_bits)) |
1182 | return -1; |
1183 | |
1184 | /* Per GDMA spec, rmb is necessary after checking owner_bits, before |
1185 | * reading completion info |
1186 | */ |
1187 | rmb(); |
1188 | |
1189 | comp->wq_num = cqe->cqe_info.wq_num; |
1190 | comp->is_sq = cqe->cqe_info.is_sq; |
1191 | memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE); |
1192 | |
1193 | return 1; |
1194 | } |
1195 | |
1196 | int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) |
1197 | { |
1198 | int cqe_idx; |
1199 | int ret; |
1200 | |
1201 | for (cqe_idx = 0; cqe_idx < num_cqe; cqe_idx++) { |
1202 | ret = mana_gd_read_cqe(cq, comp: &comp[cqe_idx]); |
1203 | |
1204 | if (ret < 0) { |
1205 | cq->head -= cqe_idx; |
1206 | return ret; |
1207 | } |
1208 | |
1209 | if (ret == 0) |
1210 | break; |
1211 | |
1212 | cq->head++; |
1213 | } |
1214 | |
1215 | return cqe_idx; |
1216 | } |
1217 | |
1218 | static irqreturn_t mana_gd_intr(int irq, void *arg) |
1219 | { |
1220 | struct gdma_irq_context *gic = arg; |
1221 | struct list_head *eq_list = &gic->eq_list; |
1222 | struct gdma_queue *eq; |
1223 | |
1224 | rcu_read_lock(); |
1225 | list_for_each_entry_rcu(eq, eq_list, entry) { |
1226 | gic->handler(eq); |
1227 | } |
1228 | rcu_read_unlock(); |
1229 | |
1230 | return IRQ_HANDLED; |
1231 | } |
1232 | |
1233 | int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r) |
1234 | { |
1235 | r->map = bitmap_zalloc(nbits: res_avail, GFP_KERNEL); |
1236 | if (!r->map) |
1237 | return -ENOMEM; |
1238 | |
1239 | r->size = res_avail; |
1240 | spin_lock_init(&r->lock); |
1241 | |
1242 | return 0; |
1243 | } |
1244 | |
1245 | void mana_gd_free_res_map(struct gdma_resource *r) |
1246 | { |
1247 | bitmap_free(bitmap: r->map); |
1248 | r->map = NULL; |
1249 | r->size = 0; |
1250 | } |
1251 | |
1252 | static int irq_setup(unsigned int *irqs, unsigned int len, int node) |
1253 | { |
1254 | const struct cpumask *next, *prev = cpu_none_mask; |
1255 | cpumask_var_t cpus __free(free_cpumask_var); |
1256 | int cpu, weight; |
1257 | |
1258 | if (!alloc_cpumask_var(mask: &cpus, GFP_KERNEL)) |
1259 | return -ENOMEM; |
1260 | |
1261 | rcu_read_lock(); |
1262 | for_each_numa_hop_mask(next, node) { |
1263 | weight = cpumask_weight_andnot(srcp1: next, srcp2: prev); |
1264 | while (weight > 0) { |
1265 | cpumask_andnot(dstp: cpus, src1p: next, src2p: prev); |
1266 | for_each_cpu(cpu, cpus) { |
1267 | if (len-- == 0) |
1268 | goto done; |
1269 | irq_set_affinity_and_hint(irq: *irqs++, topology_sibling_cpumask(cpu)); |
1270 | cpumask_andnot(dstp: cpus, src1p: cpus, topology_sibling_cpumask(cpu)); |
1271 | --weight; |
1272 | } |
1273 | } |
1274 | prev = next; |
1275 | } |
1276 | done: |
1277 | rcu_read_unlock(); |
1278 | return 0; |
1279 | } |
1280 | |
1281 | static int mana_gd_setup_irqs(struct pci_dev *pdev) |
1282 | { |
1283 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1284 | unsigned int max_queues_per_port; |
1285 | struct gdma_irq_context *gic; |
1286 | unsigned int max_irqs, cpu; |
1287 | int start_irq_index = 1; |
1288 | int nvec, *irqs, irq; |
1289 | int err, i = 0, j; |
1290 | |
1291 | cpus_read_lock(); |
1292 | max_queues_per_port = num_online_cpus(); |
1293 | if (max_queues_per_port > MANA_MAX_NUM_QUEUES) |
1294 | max_queues_per_port = MANA_MAX_NUM_QUEUES; |
1295 | |
1296 | /* Need 1 interrupt for the Hardware communication Channel (HWC) */ |
1297 | max_irqs = max_queues_per_port + 1; |
1298 | |
1299 | nvec = pci_alloc_irq_vectors(dev: pdev, min_vecs: 2, max_vecs: max_irqs, PCI_IRQ_MSIX); |
1300 | if (nvec < 0) { |
1301 | cpus_read_unlock(); |
1302 | return nvec; |
1303 | } |
1304 | if (nvec <= num_online_cpus()) |
1305 | start_irq_index = 0; |
1306 | |
1307 | irqs = kmalloc_array(n: (nvec - start_irq_index), size: sizeof(int), GFP_KERNEL); |
1308 | if (!irqs) { |
1309 | err = -ENOMEM; |
1310 | goto free_irq_vector; |
1311 | } |
1312 | |
1313 | gc->irq_contexts = kcalloc(n: nvec, size: sizeof(struct gdma_irq_context), |
1314 | GFP_KERNEL); |
1315 | if (!gc->irq_contexts) { |
1316 | err = -ENOMEM; |
1317 | goto free_irq_vector; |
1318 | } |
1319 | |
1320 | for (i = 0; i < nvec; i++) { |
1321 | gic = &gc->irq_contexts[i]; |
1322 | gic->handler = mana_gd_process_eq_events; |
1323 | INIT_LIST_HEAD(list: &gic->eq_list); |
1324 | spin_lock_init(&gic->lock); |
1325 | |
1326 | if (!i) |
1327 | snprintf(buf: gic->name, MANA_IRQ_NAME_SZ, fmt: "mana_hwc@pci:%s" , |
1328 | pci_name(pdev)); |
1329 | else |
1330 | snprintf(buf: gic->name, MANA_IRQ_NAME_SZ, fmt: "mana_q%d@pci:%s" , |
1331 | i - 1, pci_name(pdev)); |
1332 | |
1333 | irq = pci_irq_vector(dev: pdev, nr: i); |
1334 | if (irq < 0) { |
1335 | err = irq; |
1336 | goto free_irq; |
1337 | } |
1338 | |
1339 | if (!i) { |
1340 | err = request_irq(irq, handler: mana_gd_intr, flags: 0, name: gic->name, dev: gic); |
1341 | if (err) |
1342 | goto free_irq; |
1343 | |
1344 | /* If number of IRQ is one extra than number of online CPUs, |
1345 | * then we need to assign IRQ0 (hwc irq) and IRQ1 to |
1346 | * same CPU. |
1347 | * Else we will use different CPUs for IRQ0 and IRQ1. |
1348 | * Also we are using cpumask_local_spread instead of |
1349 | * cpumask_first for the node, because the node can be |
1350 | * mem only. |
1351 | */ |
1352 | if (start_irq_index) { |
1353 | cpu = cpumask_local_spread(i, node: gc->numa_node); |
1354 | irq_set_affinity_and_hint(irq, cpumask_of(cpu)); |
1355 | } else { |
1356 | irqs[start_irq_index] = irq; |
1357 | } |
1358 | } else { |
1359 | irqs[i - start_irq_index] = irq; |
1360 | err = request_irq(irq: irqs[i - start_irq_index], handler: mana_gd_intr, flags: 0, |
1361 | name: gic->name, dev: gic); |
1362 | if (err) |
1363 | goto free_irq; |
1364 | } |
1365 | } |
1366 | |
1367 | err = irq_setup(irqs, len: (nvec - start_irq_index), node: gc->numa_node); |
1368 | if (err) |
1369 | goto free_irq; |
1370 | |
1371 | gc->max_num_msix = nvec; |
1372 | gc->num_msix_usable = nvec; |
1373 | cpus_read_unlock(); |
1374 | return 0; |
1375 | |
1376 | free_irq: |
1377 | for (j = i - 1; j >= 0; j--) { |
1378 | irq = pci_irq_vector(dev: pdev, nr: j); |
1379 | gic = &gc->irq_contexts[j]; |
1380 | |
1381 | irq_update_affinity_hint(irq, NULL); |
1382 | free_irq(irq, gic); |
1383 | } |
1384 | |
1385 | kfree(objp: gc->irq_contexts); |
1386 | kfree(objp: irqs); |
1387 | gc->irq_contexts = NULL; |
1388 | free_irq_vector: |
1389 | cpus_read_unlock(); |
1390 | pci_free_irq_vectors(dev: pdev); |
1391 | return err; |
1392 | } |
1393 | |
1394 | static void mana_gd_remove_irqs(struct pci_dev *pdev) |
1395 | { |
1396 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1397 | struct gdma_irq_context *gic; |
1398 | int irq, i; |
1399 | |
1400 | if (gc->max_num_msix < 1) |
1401 | return; |
1402 | |
1403 | for (i = 0; i < gc->max_num_msix; i++) { |
1404 | irq = pci_irq_vector(dev: pdev, nr: i); |
1405 | if (irq < 0) |
1406 | continue; |
1407 | |
1408 | gic = &gc->irq_contexts[i]; |
1409 | |
1410 | /* Need to clear the hint before free_irq */ |
1411 | irq_update_affinity_hint(irq, NULL); |
1412 | free_irq(irq, gic); |
1413 | } |
1414 | |
1415 | pci_free_irq_vectors(dev: pdev); |
1416 | |
1417 | gc->max_num_msix = 0; |
1418 | gc->num_msix_usable = 0; |
1419 | kfree(objp: gc->irq_contexts); |
1420 | gc->irq_contexts = NULL; |
1421 | } |
1422 | |
1423 | static int mana_gd_setup(struct pci_dev *pdev) |
1424 | { |
1425 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1426 | int err; |
1427 | |
1428 | mana_gd_init_registers(pdev); |
1429 | mana_smc_init(sc: &gc->shm_channel, dev: gc->dev, base: gc->shm_base); |
1430 | |
1431 | err = mana_gd_setup_irqs(pdev); |
1432 | if (err) |
1433 | return err; |
1434 | |
1435 | err = mana_hwc_create_channel(gc); |
1436 | if (err) |
1437 | goto remove_irq; |
1438 | |
1439 | err = mana_gd_verify_vf_version(pdev); |
1440 | if (err) |
1441 | goto destroy_hwc; |
1442 | |
1443 | err = mana_gd_query_max_resources(pdev); |
1444 | if (err) |
1445 | goto destroy_hwc; |
1446 | |
1447 | err = mana_gd_detect_devices(pdev); |
1448 | if (err) |
1449 | goto destroy_hwc; |
1450 | |
1451 | return 0; |
1452 | |
1453 | destroy_hwc: |
1454 | mana_hwc_destroy_channel(gc); |
1455 | remove_irq: |
1456 | mana_gd_remove_irqs(pdev); |
1457 | return err; |
1458 | } |
1459 | |
1460 | static void mana_gd_cleanup(struct pci_dev *pdev) |
1461 | { |
1462 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1463 | |
1464 | mana_hwc_destroy_channel(gc); |
1465 | |
1466 | mana_gd_remove_irqs(pdev); |
1467 | } |
1468 | |
1469 | static bool mana_is_pf(unsigned short dev_id) |
1470 | { |
1471 | return dev_id == MANA_PF_DEVICE_ID; |
1472 | } |
1473 | |
1474 | static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) |
1475 | { |
1476 | struct gdma_context *gc; |
1477 | void __iomem *bar0_va; |
1478 | int bar = 0; |
1479 | int err; |
1480 | |
1481 | /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ |
1482 | BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); |
1483 | |
1484 | err = pci_enable_device(dev: pdev); |
1485 | if (err) |
1486 | return -ENXIO; |
1487 | |
1488 | pci_set_master(dev: pdev); |
1489 | |
1490 | err = pci_request_regions(pdev, "mana" ); |
1491 | if (err) |
1492 | goto disable_dev; |
1493 | |
1494 | err = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
1495 | if (err) |
1496 | goto release_region; |
1497 | |
1498 | err = dma_set_max_seg_size(dev: &pdev->dev, UINT_MAX); |
1499 | if (err) { |
1500 | dev_err(&pdev->dev, "Failed to set dma device segment size\n" ); |
1501 | goto release_region; |
1502 | } |
1503 | |
1504 | err = -ENOMEM; |
1505 | gc = vzalloc(size: sizeof(*gc)); |
1506 | if (!gc) |
1507 | goto release_region; |
1508 | |
1509 | mutex_init(&gc->eq_test_event_mutex); |
1510 | pci_set_drvdata(pdev, data: gc); |
1511 | gc->bar0_pa = pci_resource_start(pdev, 0); |
1512 | |
1513 | bar0_va = pci_iomap(dev: pdev, bar, max: 0); |
1514 | if (!bar0_va) |
1515 | goto free_gc; |
1516 | |
1517 | gc->numa_node = dev_to_node(dev: &pdev->dev); |
1518 | gc->is_pf = mana_is_pf(dev_id: pdev->device); |
1519 | gc->bar0_va = bar0_va; |
1520 | gc->dev = &pdev->dev; |
1521 | |
1522 | err = mana_gd_setup(pdev); |
1523 | if (err) |
1524 | goto unmap_bar; |
1525 | |
1526 | err = mana_probe(gd: &gc->mana, resuming: false); |
1527 | if (err) |
1528 | goto cleanup_gd; |
1529 | |
1530 | return 0; |
1531 | |
1532 | cleanup_gd: |
1533 | mana_gd_cleanup(pdev); |
1534 | unmap_bar: |
1535 | pci_iounmap(dev: pdev, bar0_va); |
1536 | free_gc: |
1537 | pci_set_drvdata(pdev, NULL); |
1538 | vfree(addr: gc); |
1539 | release_region: |
1540 | pci_release_regions(pdev); |
1541 | disable_dev: |
1542 | pci_disable_device(dev: pdev); |
1543 | dev_err(&pdev->dev, "gdma probe failed: err = %d\n" , err); |
1544 | return err; |
1545 | } |
1546 | |
1547 | static void mana_gd_remove(struct pci_dev *pdev) |
1548 | { |
1549 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1550 | |
1551 | mana_remove(gd: &gc->mana, suspending: false); |
1552 | |
1553 | mana_gd_cleanup(pdev); |
1554 | |
1555 | pci_iounmap(dev: pdev, gc->bar0_va); |
1556 | |
1557 | vfree(addr: gc); |
1558 | |
1559 | pci_release_regions(pdev); |
1560 | pci_disable_device(dev: pdev); |
1561 | } |
1562 | |
1563 | /* The 'state' parameter is not used. */ |
1564 | static int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) |
1565 | { |
1566 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1567 | |
1568 | mana_remove(gd: &gc->mana, suspending: true); |
1569 | |
1570 | mana_gd_cleanup(pdev); |
1571 | |
1572 | return 0; |
1573 | } |
1574 | |
1575 | /* In case the NIC hardware stops working, the suspend and resume callbacks will |
1576 | * fail -- if this happens, it's safer to just report an error than try to undo |
1577 | * what has been done. |
1578 | */ |
1579 | static int mana_gd_resume(struct pci_dev *pdev) |
1580 | { |
1581 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1582 | int err; |
1583 | |
1584 | err = mana_gd_setup(pdev); |
1585 | if (err) |
1586 | return err; |
1587 | |
1588 | err = mana_probe(gd: &gc->mana, resuming: true); |
1589 | if (err) |
1590 | return err; |
1591 | |
1592 | return 0; |
1593 | } |
1594 | |
1595 | /* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ |
1596 | static void mana_gd_shutdown(struct pci_dev *pdev) |
1597 | { |
1598 | struct gdma_context *gc = pci_get_drvdata(pdev); |
1599 | |
1600 | dev_info(&pdev->dev, "Shutdown was called\n" ); |
1601 | |
1602 | mana_remove(gd: &gc->mana, suspending: true); |
1603 | |
1604 | mana_gd_cleanup(pdev); |
1605 | |
1606 | pci_disable_device(dev: pdev); |
1607 | } |
1608 | |
1609 | static const struct pci_device_id mana_id_table[] = { |
1610 | { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, |
1611 | { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, |
1612 | { } |
1613 | }; |
1614 | |
1615 | static struct pci_driver mana_driver = { |
1616 | .name = "mana" , |
1617 | .id_table = mana_id_table, |
1618 | .probe = mana_gd_probe, |
1619 | .remove = mana_gd_remove, |
1620 | .suspend = mana_gd_suspend, |
1621 | .resume = mana_gd_resume, |
1622 | .shutdown = mana_gd_shutdown, |
1623 | }; |
1624 | |
1625 | module_pci_driver(mana_driver); |
1626 | |
1627 | MODULE_DEVICE_TABLE(pci, mana_id_table); |
1628 | |
1629 | MODULE_LICENSE("Dual BSD/GPL" ); |
1630 | MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver" ); |
1631 | |