1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) |
2 | /* QLogic qed NIC Driver |
3 | * Copyright (c) 2015-2017 QLogic Corporation |
4 | * Copyright (c) 2019-2020 Marvell International Ltd. |
5 | */ |
6 | |
7 | #include <linux/types.h> |
8 | #include <linux/dma-mapping.h> |
9 | #include <linux/kernel.h> |
10 | #include <linux/list.h> |
11 | #include <linux/pci.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/string.h> |
14 | #include "qed.h" |
15 | #include "qed_iscsi.h" |
16 | #include "qed_ll2.h" |
17 | #include "qed_ooo.h" |
18 | #include "qed_cxt.h" |
19 | #include "qed_nvmetcp.h" |
20 | static struct qed_ooo_archipelago |
21 | *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn, |
22 | struct qed_ooo_info |
23 | *p_ooo_info, |
24 | u32 cid) |
25 | { |
26 | u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; |
27 | struct qed_ooo_archipelago *p_archipelago; |
28 | |
29 | if (unlikely(idx >= p_ooo_info->max_num_archipelagos)) |
30 | return NULL; |
31 | |
32 | p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; |
33 | |
34 | if (unlikely(list_empty(&p_archipelago->isles_list))) |
35 | return NULL; |
36 | |
37 | return p_archipelago; |
38 | } |
39 | |
40 | static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn, |
41 | struct qed_ooo_info *p_ooo_info, |
42 | u32 cid, u8 isle) |
43 | { |
44 | struct qed_ooo_archipelago *p_archipelago = NULL; |
45 | struct qed_ooo_isle *p_isle = NULL; |
46 | u8 the_num_of_isle = 1; |
47 | |
48 | p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); |
49 | if (unlikely(!p_archipelago)) { |
50 | DP_NOTICE(p_hwfn, |
51 | "Connection %d is not found in OOO list\n" , cid); |
52 | return NULL; |
53 | } |
54 | |
55 | list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) { |
56 | if (the_num_of_isle == isle) |
57 | return p_isle; |
58 | the_num_of_isle++; |
59 | } |
60 | |
61 | return NULL; |
62 | } |
63 | |
64 | void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn, |
65 | struct qed_ooo_info *p_ooo_info, |
66 | struct ooo_opaque *p_cqe) |
67 | { |
68 | struct qed_ooo_history *p_history = &p_ooo_info->ooo_history; |
69 | |
70 | if (p_history->head_idx == p_history->num_of_cqes) |
71 | p_history->head_idx = 0; |
72 | p_history->p_cqes[p_history->head_idx] = *p_cqe; |
73 | p_history->head_idx++; |
74 | } |
75 | |
76 | int qed_ooo_alloc(struct qed_hwfn *p_hwfn) |
77 | { |
78 | u16 max_num_archipelagos = 0, cid_base; |
79 | struct qed_ooo_info *p_ooo_info; |
80 | enum protocol_type proto; |
81 | u16 max_num_isles = 0; |
82 | u32 i; |
83 | |
84 | switch (p_hwfn->hw_info.personality) { |
85 | case QED_PCI_ISCSI: |
86 | case QED_PCI_NVMETCP: |
87 | proto = PROTOCOLID_TCP_ULP; |
88 | break; |
89 | case QED_PCI_ETH_RDMA: |
90 | case QED_PCI_ETH_IWARP: |
91 | proto = PROTOCOLID_IWARP; |
92 | break; |
93 | default: |
94 | DP_NOTICE(p_hwfn, |
95 | "Failed to allocate qed_ooo_info: unknown personality\n" ); |
96 | return -EINVAL; |
97 | } |
98 | |
99 | max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, type: proto, |
100 | NULL); |
101 | max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos; |
102 | cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, type: proto); |
103 | |
104 | if (!max_num_archipelagos) { |
105 | DP_NOTICE(p_hwfn, |
106 | "Failed to allocate qed_ooo_info: unknown amount of connections\n" ); |
107 | return -EINVAL; |
108 | } |
109 | |
110 | p_ooo_info = kzalloc(size: sizeof(*p_ooo_info), GFP_KERNEL); |
111 | if (!p_ooo_info) |
112 | return -ENOMEM; |
113 | |
114 | p_ooo_info->cid_base = cid_base; |
115 | p_ooo_info->max_num_archipelagos = max_num_archipelagos; |
116 | |
117 | INIT_LIST_HEAD(list: &p_ooo_info->free_buffers_list); |
118 | INIT_LIST_HEAD(list: &p_ooo_info->ready_buffers_list); |
119 | INIT_LIST_HEAD(list: &p_ooo_info->free_isles_list); |
120 | |
121 | p_ooo_info->p_isles_mem = kcalloc(n: max_num_isles, |
122 | size: sizeof(struct qed_ooo_isle), |
123 | GFP_KERNEL); |
124 | if (!p_ooo_info->p_isles_mem) |
125 | goto no_isles_mem; |
126 | |
127 | for (i = 0; i < max_num_isles; i++) { |
128 | INIT_LIST_HEAD(list: &p_ooo_info->p_isles_mem[i].buffers_list); |
129 | list_add_tail(new: &p_ooo_info->p_isles_mem[i].list_entry, |
130 | head: &p_ooo_info->free_isles_list); |
131 | } |
132 | |
133 | p_ooo_info->p_archipelagos_mem = |
134 | kcalloc(n: max_num_archipelagos, |
135 | size: sizeof(struct qed_ooo_archipelago), |
136 | GFP_KERNEL); |
137 | if (!p_ooo_info->p_archipelagos_mem) |
138 | goto no_archipelagos_mem; |
139 | |
140 | for (i = 0; i < max_num_archipelagos; i++) |
141 | INIT_LIST_HEAD(list: &p_ooo_info->p_archipelagos_mem[i].isles_list); |
142 | |
143 | p_ooo_info->ooo_history.p_cqes = |
144 | kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES, |
145 | size: sizeof(struct ooo_opaque), |
146 | GFP_KERNEL); |
147 | if (!p_ooo_info->ooo_history.p_cqes) |
148 | goto no_history_mem; |
149 | |
150 | p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES; |
151 | |
152 | p_hwfn->p_ooo_info = p_ooo_info; |
153 | return 0; |
154 | |
155 | no_history_mem: |
156 | kfree(objp: p_ooo_info->p_archipelagos_mem); |
157 | no_archipelagos_mem: |
158 | kfree(objp: p_ooo_info->p_isles_mem); |
159 | no_isles_mem: |
160 | kfree(objp: p_ooo_info); |
161 | return -ENOMEM; |
162 | } |
163 | |
164 | void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn, |
165 | struct qed_ooo_info *p_ooo_info, u32 cid) |
166 | { |
167 | struct qed_ooo_archipelago *p_archipelago; |
168 | struct qed_ooo_buffer *p_buffer; |
169 | struct qed_ooo_isle *p_isle; |
170 | |
171 | p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); |
172 | if (!p_archipelago) |
173 | return; |
174 | |
175 | while (!list_empty(head: &p_archipelago->isles_list)) { |
176 | p_isle = list_first_entry(&p_archipelago->isles_list, |
177 | struct qed_ooo_isle, list_entry); |
178 | |
179 | list_del(entry: &p_isle->list_entry); |
180 | |
181 | while (!list_empty(head: &p_isle->buffers_list)) { |
182 | p_buffer = list_first_entry(&p_isle->buffers_list, |
183 | struct qed_ooo_buffer, |
184 | list_entry); |
185 | |
186 | if (!p_buffer) |
187 | break; |
188 | |
189 | list_move_tail(list: &p_buffer->list_entry, |
190 | head: &p_ooo_info->free_buffers_list); |
191 | } |
192 | list_add_tail(new: &p_isle->list_entry, |
193 | head: &p_ooo_info->free_isles_list); |
194 | } |
195 | } |
196 | |
197 | void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn, |
198 | struct qed_ooo_info *p_ooo_info) |
199 | { |
200 | struct qed_ooo_archipelago *p_archipelago; |
201 | struct qed_ooo_buffer *p_buffer; |
202 | struct qed_ooo_isle *p_isle; |
203 | u32 i; |
204 | |
205 | for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) { |
206 | p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]); |
207 | |
208 | while (!list_empty(head: &p_archipelago->isles_list)) { |
209 | p_isle = list_first_entry(&p_archipelago->isles_list, |
210 | struct qed_ooo_isle, |
211 | list_entry); |
212 | |
213 | list_del(entry: &p_isle->list_entry); |
214 | |
215 | while (!list_empty(head: &p_isle->buffers_list)) { |
216 | p_buffer = |
217 | list_first_entry(&p_isle->buffers_list, |
218 | struct qed_ooo_buffer, |
219 | list_entry); |
220 | |
221 | if (!p_buffer) |
222 | break; |
223 | |
224 | list_move_tail(list: &p_buffer->list_entry, |
225 | head: &p_ooo_info->free_buffers_list); |
226 | } |
227 | list_add_tail(new: &p_isle->list_entry, |
228 | head: &p_ooo_info->free_isles_list); |
229 | } |
230 | } |
231 | if (!list_empty(head: &p_ooo_info->ready_buffers_list)) |
232 | list_splice_tail_init(list: &p_ooo_info->ready_buffers_list, |
233 | head: &p_ooo_info->free_buffers_list); |
234 | } |
235 | |
236 | void qed_ooo_setup(struct qed_hwfn *p_hwfn) |
237 | { |
238 | qed_ooo_release_all_isles(p_hwfn, p_ooo_info: p_hwfn->p_ooo_info); |
239 | memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0, |
240 | p_hwfn->p_ooo_info->ooo_history.num_of_cqes * |
241 | sizeof(struct ooo_opaque)); |
242 | p_hwfn->p_ooo_info->ooo_history.head_idx = 0; |
243 | } |
244 | |
245 | void qed_ooo_free(struct qed_hwfn *p_hwfn) |
246 | { |
247 | struct qed_ooo_info *p_ooo_info = p_hwfn->p_ooo_info; |
248 | struct qed_ooo_buffer *p_buffer; |
249 | |
250 | if (!p_ooo_info) |
251 | return; |
252 | |
253 | qed_ooo_release_all_isles(p_hwfn, p_ooo_info); |
254 | while (!list_empty(head: &p_ooo_info->free_buffers_list)) { |
255 | p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, |
256 | struct qed_ooo_buffer, list_entry); |
257 | |
258 | if (!p_buffer) |
259 | break; |
260 | |
261 | list_del(entry: &p_buffer->list_entry); |
262 | dma_free_coherent(dev: &p_hwfn->cdev->pdev->dev, |
263 | size: p_buffer->rx_buffer_size, |
264 | cpu_addr: p_buffer->rx_buffer_virt_addr, |
265 | dma_handle: p_buffer->rx_buffer_phys_addr); |
266 | kfree(objp: p_buffer); |
267 | } |
268 | |
269 | kfree(objp: p_ooo_info->p_isles_mem); |
270 | kfree(objp: p_ooo_info->p_archipelagos_mem); |
271 | kfree(objp: p_ooo_info->ooo_history.p_cqes); |
272 | kfree(objp: p_ooo_info); |
273 | p_hwfn->p_ooo_info = NULL; |
274 | } |
275 | |
276 | void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn, |
277 | struct qed_ooo_info *p_ooo_info, |
278 | struct qed_ooo_buffer *p_buffer) |
279 | { |
280 | list_add_tail(new: &p_buffer->list_entry, head: &p_ooo_info->free_buffers_list); |
281 | } |
282 | |
283 | struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn, |
284 | struct qed_ooo_info *p_ooo_info) |
285 | { |
286 | struct qed_ooo_buffer *p_buffer = NULL; |
287 | |
288 | if (!list_empty(head: &p_ooo_info->free_buffers_list)) { |
289 | p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, |
290 | struct qed_ooo_buffer, list_entry); |
291 | |
292 | list_del(entry: &p_buffer->list_entry); |
293 | } |
294 | |
295 | return p_buffer; |
296 | } |
297 | |
298 | void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn, |
299 | struct qed_ooo_info *p_ooo_info, |
300 | struct qed_ooo_buffer *p_buffer, u8 on_tail) |
301 | { |
302 | if (on_tail) |
303 | list_add_tail(new: &p_buffer->list_entry, |
304 | head: &p_ooo_info->ready_buffers_list); |
305 | else |
306 | list_add(new: &p_buffer->list_entry, |
307 | head: &p_ooo_info->ready_buffers_list); |
308 | } |
309 | |
310 | struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn, |
311 | struct qed_ooo_info *p_ooo_info) |
312 | { |
313 | struct qed_ooo_buffer *p_buffer = NULL; |
314 | |
315 | if (!list_empty(head: &p_ooo_info->ready_buffers_list)) { |
316 | p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list, |
317 | struct qed_ooo_buffer, list_entry); |
318 | |
319 | list_del(entry: &p_buffer->list_entry); |
320 | } |
321 | |
322 | return p_buffer; |
323 | } |
324 | |
325 | void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn, |
326 | struct qed_ooo_info *p_ooo_info, |
327 | u32 cid, u8 drop_isle, u8 drop_size) |
328 | { |
329 | struct qed_ooo_isle *p_isle = NULL; |
330 | u8 isle_idx; |
331 | |
332 | for (isle_idx = 0; isle_idx < drop_size; isle_idx++) { |
333 | p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, isle: drop_isle); |
334 | if (!p_isle) { |
335 | DP_NOTICE(p_hwfn, |
336 | "Isle %d is not found(cid %d)\n" , |
337 | drop_isle, cid); |
338 | return; |
339 | } |
340 | if (list_empty(head: &p_isle->buffers_list)) |
341 | DP_NOTICE(p_hwfn, |
342 | "Isle %d is empty(cid %d)\n" , drop_isle, cid); |
343 | else |
344 | list_splice_tail_init(list: &p_isle->buffers_list, |
345 | head: &p_ooo_info->free_buffers_list); |
346 | |
347 | list_del(entry: &p_isle->list_entry); |
348 | p_ooo_info->cur_isles_number--; |
349 | list_add(new: &p_isle->list_entry, head: &p_ooo_info->free_isles_list); |
350 | } |
351 | } |
352 | |
353 | void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn, |
354 | struct qed_ooo_info *p_ooo_info, |
355 | u32 cid, u8 ooo_isle, |
356 | struct qed_ooo_buffer *p_buffer) |
357 | { |
358 | struct qed_ooo_archipelago *p_archipelago = NULL; |
359 | struct qed_ooo_isle *p_prev_isle = NULL; |
360 | struct qed_ooo_isle *p_isle = NULL; |
361 | |
362 | if (ooo_isle > 1) { |
363 | p_prev_isle = qed_ooo_seek_isle(p_hwfn, |
364 | p_ooo_info, cid, isle: ooo_isle - 1); |
365 | if (unlikely(!p_prev_isle)) { |
366 | DP_NOTICE(p_hwfn, |
367 | "Isle %d is not found(cid %d)\n" , |
368 | ooo_isle - 1, cid); |
369 | return; |
370 | } |
371 | } |
372 | p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); |
373 | if (unlikely(!p_archipelago && ooo_isle != 1)) { |
374 | DP_NOTICE(p_hwfn, |
375 | "Connection %d is not found in OOO list\n" , cid); |
376 | return; |
377 | } |
378 | |
379 | if (!list_empty(head: &p_ooo_info->free_isles_list)) { |
380 | p_isle = list_first_entry(&p_ooo_info->free_isles_list, |
381 | struct qed_ooo_isle, list_entry); |
382 | |
383 | list_del(entry: &p_isle->list_entry); |
384 | if (unlikely(!list_empty(&p_isle->buffers_list))) { |
385 | DP_NOTICE(p_hwfn, "Free isle is not empty\n" ); |
386 | INIT_LIST_HEAD(list: &p_isle->buffers_list); |
387 | } |
388 | } else { |
389 | DP_NOTICE(p_hwfn, "No more free isles\n" ); |
390 | return; |
391 | } |
392 | |
393 | if (!p_archipelago) { |
394 | u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; |
395 | |
396 | p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; |
397 | } |
398 | |
399 | list_add(new: &p_buffer->list_entry, head: &p_isle->buffers_list); |
400 | p_ooo_info->cur_isles_number++; |
401 | p_ooo_info->gen_isles_number++; |
402 | |
403 | if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number) |
404 | p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number; |
405 | |
406 | if (!p_prev_isle) |
407 | list_add(new: &p_isle->list_entry, head: &p_archipelago->isles_list); |
408 | else |
409 | list_add(new: &p_isle->list_entry, head: &p_prev_isle->list_entry); |
410 | } |
411 | |
412 | void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn, |
413 | struct qed_ooo_info *p_ooo_info, |
414 | u32 cid, |
415 | u8 ooo_isle, |
416 | struct qed_ooo_buffer *p_buffer, u8 buffer_side) |
417 | { |
418 | struct qed_ooo_isle *p_isle = NULL; |
419 | |
420 | p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, isle: ooo_isle); |
421 | if (unlikely(!p_isle)) { |
422 | DP_NOTICE(p_hwfn, |
423 | "Isle %d is not found(cid %d)\n" , ooo_isle, cid); |
424 | return; |
425 | } |
426 | |
427 | if (unlikely(buffer_side == QED_OOO_LEFT_BUF)) |
428 | list_add(new: &p_buffer->list_entry, head: &p_isle->buffers_list); |
429 | else |
430 | list_add_tail(new: &p_buffer->list_entry, head: &p_isle->buffers_list); |
431 | } |
432 | |
433 | void qed_ooo_join_isles(struct qed_hwfn *p_hwfn, |
434 | struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle) |
435 | { |
436 | struct qed_ooo_isle *p_right_isle = NULL; |
437 | struct qed_ooo_isle *p_left_isle = NULL; |
438 | |
439 | p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, |
440 | isle: left_isle + 1); |
441 | if (unlikely(!p_right_isle)) { |
442 | DP_NOTICE(p_hwfn, |
443 | "Right isle %d is not found(cid %d)\n" , |
444 | left_isle + 1, cid); |
445 | return; |
446 | } |
447 | |
448 | list_del(entry: &p_right_isle->list_entry); |
449 | p_ooo_info->cur_isles_number--; |
450 | if (left_isle) { |
451 | p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, |
452 | isle: left_isle); |
453 | if (unlikely(!p_left_isle)) { |
454 | DP_NOTICE(p_hwfn, |
455 | "Left isle %d is not found(cid %d)\n" , |
456 | left_isle, cid); |
457 | return; |
458 | } |
459 | list_splice_tail_init(list: &p_right_isle->buffers_list, |
460 | head: &p_left_isle->buffers_list); |
461 | } else { |
462 | list_splice_tail_init(list: &p_right_isle->buffers_list, |
463 | head: &p_ooo_info->ready_buffers_list); |
464 | } |
465 | list_add_tail(new: &p_right_isle->list_entry, head: &p_ooo_info->free_isles_list); |
466 | } |
467 | |