1/*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include "mlx5_ib.h"
34
35struct mlx5_ib_gsi_wr {
36 struct ib_cqe cqe;
37 struct ib_wc wc;
38 bool completed:1;
39};
40
41static bool mlx5_ib_deth_sqpn_cap(struct mlx5_ib_dev *dev)
42{
43 return MLX5_CAP_GEN(dev->mdev, set_deth_sqpn);
44}
45
46/* Call with gsi->lock locked */
47static void generate_completions(struct mlx5_ib_qp *mqp)
48{
49 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
50 struct ib_cq *gsi_cq = mqp->ibqp.send_cq;
51 struct mlx5_ib_gsi_wr *wr;
52 u32 index;
53
54 for (index = gsi->outstanding_ci; index != gsi->outstanding_pi;
55 index++) {
56 wr = &gsi->outstanding_wrs[index % gsi->cap.max_send_wr];
57
58 if (!wr->completed)
59 break;
60
61 WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &wr->wc));
62 wr->completed = false;
63 }
64
65 gsi->outstanding_ci = index;
66}
67
68static void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc)
69{
70 struct mlx5_ib_gsi_qp *gsi = cq->cq_context;
71 struct mlx5_ib_gsi_wr *wr =
72 container_of(wc->wr_cqe, struct mlx5_ib_gsi_wr, cqe);
73 struct mlx5_ib_qp *mqp = container_of(gsi, struct mlx5_ib_qp, gsi);
74 u64 wr_id;
75 unsigned long flags;
76
77 spin_lock_irqsave(&gsi->lock, flags);
78 wr->completed = true;
79 wr_id = wr->wc.wr_id;
80 wr->wc = *wc;
81 wr->wc.wr_id = wr_id;
82 wr->wc.qp = &mqp->ibqp;
83
84 generate_completions(mqp);
85 spin_unlock_irqrestore(lock: &gsi->lock, flags);
86}
87
88int mlx5_ib_create_gsi(struct ib_pd *pd, struct mlx5_ib_qp *mqp,
89 struct ib_qp_init_attr *attr)
90{
91 struct mlx5_ib_dev *dev = to_mdev(ibdev: pd->device);
92 struct mlx5_ib_gsi_qp *gsi;
93 struct ib_qp_init_attr hw_init_attr = *attr;
94 const u8 port_num = attr->port_num;
95 int num_qps = 0;
96 int ret;
97
98 if (mlx5_ib_deth_sqpn_cap(dev)) {
99 if (MLX5_CAP_GEN(dev->mdev,
100 port_type) == MLX5_CAP_PORT_TYPE_IB)
101 num_qps = pd->device->attrs.max_pkeys;
102 else if (dev->lag_active)
103 num_qps = dev->lag_ports;
104 }
105
106 gsi = &mqp->gsi;
107 gsi->tx_qps = kcalloc(num_qps, sizeof(*gsi->tx_qps), GFP_KERNEL);
108 if (!gsi->tx_qps)
109 return -ENOMEM;
110
111 gsi->outstanding_wrs =
112 kcalloc(attr->cap.max_send_wr, sizeof(*gsi->outstanding_wrs),
113 GFP_KERNEL);
114 if (!gsi->outstanding_wrs) {
115 ret = -ENOMEM;
116 goto err_free_tx;
117 }
118
119 if (dev->devr.ports[port_num - 1].gsi) {
120 mlx5_ib_warn(dev, "GSI QP already exists on port %d\n",
121 port_num);
122 ret = -EBUSY;
123 goto err_free_wrs;
124 }
125 gsi->num_qps = num_qps;
126 spin_lock_init(&gsi->lock);
127
128 gsi->cap = attr->cap;
129 gsi->port_num = port_num;
130
131 gsi->cq = ib_alloc_cq(dev: pd->device, private: gsi, nr_cqe: attr->cap.max_send_wr, comp_vector: 0,
132 poll_ctx: IB_POLL_SOFTIRQ);
133 if (IS_ERR(ptr: gsi->cq)) {
134 mlx5_ib_warn(dev,
135 "unable to create send CQ for GSI QP. error %pe\n",
136 gsi->cq);
137 ret = PTR_ERR(ptr: gsi->cq);
138 goto err_free_wrs;
139 }
140
141 hw_init_attr.qp_type = MLX5_IB_QPT_HW_GSI;
142 hw_init_attr.send_cq = gsi->cq;
143 if (num_qps) {
144 hw_init_attr.cap.max_send_wr = 0;
145 hw_init_attr.cap.max_send_sge = 0;
146 hw_init_attr.cap.max_inline_data = 0;
147 }
148
149 gsi->rx_qp = ib_create_qp(pd, init_attr: &hw_init_attr);
150 if (IS_ERR(ptr: gsi->rx_qp)) {
151 mlx5_ib_warn(dev,
152 "unable to create hardware GSI QP. error %pe\n",
153 gsi->rx_qp);
154 ret = PTR_ERR(ptr: gsi->rx_qp);
155 goto err_destroy_cq;
156 }
157
158 dev->devr.ports[attr->port_num - 1].gsi = gsi;
159 return 0;
160
161err_destroy_cq:
162 ib_free_cq(cq: gsi->cq);
163err_free_wrs:
164 kfree(objp: gsi->outstanding_wrs);
165err_free_tx:
166 kfree(objp: gsi->tx_qps);
167 return ret;
168}
169
170int mlx5_ib_destroy_gsi(struct mlx5_ib_qp *mqp)
171{
172 struct mlx5_ib_dev *dev = to_mdev(ibdev: mqp->ibqp.device);
173 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
174 const int port_num = gsi->port_num;
175 int qp_index;
176 int ret;
177
178 ret = ib_destroy_qp(qp: gsi->rx_qp);
179 if (ret) {
180 mlx5_ib_warn(dev, "unable to destroy hardware GSI QP. error %d\n",
181 ret);
182 return ret;
183 }
184 dev->devr.ports[port_num - 1].gsi = NULL;
185 gsi->rx_qp = NULL;
186
187 for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index) {
188 if (!gsi->tx_qps[qp_index])
189 continue;
190 WARN_ON_ONCE(ib_destroy_qp(gsi->tx_qps[qp_index]));
191 gsi->tx_qps[qp_index] = NULL;
192 }
193
194 ib_free_cq(cq: gsi->cq);
195
196 kfree(objp: gsi->outstanding_wrs);
197 kfree(objp: gsi->tx_qps);
198 return 0;
199}
200
201static struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi)
202{
203 struct ib_pd *pd = gsi->rx_qp->pd;
204 struct ib_qp_init_attr init_attr = {
205 .event_handler = gsi->rx_qp->event_handler,
206 .qp_context = gsi->rx_qp->qp_context,
207 .send_cq = gsi->cq,
208 .recv_cq = gsi->rx_qp->recv_cq,
209 .cap = {
210 .max_send_wr = gsi->cap.max_send_wr,
211 .max_send_sge = gsi->cap.max_send_sge,
212 .max_inline_data = gsi->cap.max_inline_data,
213 },
214 .qp_type = IB_QPT_UD,
215 .create_flags = MLX5_IB_QP_CREATE_SQPN_QP1,
216 };
217
218 return ib_create_qp(pd, init_attr: &init_attr);
219}
220
221static int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp,
222 u16 pkey_index)
223{
224 struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
225 struct ib_qp_attr attr;
226 int mask;
227 int ret;
228
229 mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY | IB_QP_PORT;
230 attr.qp_state = IB_QPS_INIT;
231 attr.pkey_index = pkey_index;
232 attr.qkey = IB_QP1_QKEY;
233 attr.port_num = gsi->port_num;
234 ret = ib_modify_qp(qp, qp_attr: &attr, qp_attr_mask: mask);
235 if (ret) {
236 mlx5_ib_err(dev, "could not change QP%d state to INIT: %d\n",
237 qp->qp_num, ret);
238 return ret;
239 }
240
241 attr.qp_state = IB_QPS_RTR;
242 ret = ib_modify_qp(qp, qp_attr: &attr, qp_attr_mask: IB_QP_STATE);
243 if (ret) {
244 mlx5_ib_err(dev, "could not change QP%d state to RTR: %d\n",
245 qp->qp_num, ret);
246 return ret;
247 }
248
249 attr.qp_state = IB_QPS_RTS;
250 attr.sq_psn = 0;
251 ret = ib_modify_qp(qp, qp_attr: &attr, qp_attr_mask: IB_QP_STATE | IB_QP_SQ_PSN);
252 if (ret) {
253 mlx5_ib_err(dev, "could not change QP%d state to RTS: %d\n",
254 qp->qp_num, ret);
255 return ret;
256 }
257
258 return 0;
259}
260
261static void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index)
262{
263 struct ib_device *device = gsi->rx_qp->device;
264 struct mlx5_ib_dev *dev = to_mdev(ibdev: device);
265 int pkey_index = qp_index;
266 struct mlx5_ib_qp *mqp;
267 struct ib_qp *qp;
268 unsigned long flags;
269 u16 pkey;
270 int ret;
271
272 if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
273 pkey_index = 0;
274
275 ret = ib_query_pkey(device, port_num: gsi->port_num, index: pkey_index, pkey: &pkey);
276 if (ret) {
277 mlx5_ib_warn(dev, "unable to read P_Key at port %d, index %d\n",
278 gsi->port_num, qp_index);
279 return;
280 }
281
282 if (!pkey) {
283 mlx5_ib_dbg(dev, "invalid P_Key at port %d, index %d. Skipping.\n",
284 gsi->port_num, qp_index);
285 return;
286 }
287
288 spin_lock_irqsave(&gsi->lock, flags);
289 qp = gsi->tx_qps[qp_index];
290 spin_unlock_irqrestore(lock: &gsi->lock, flags);
291 if (qp) {
292 mlx5_ib_dbg(dev, "already existing GSI TX QP at port %d, index %d. Skipping\n",
293 gsi->port_num, qp_index);
294 return;
295 }
296
297 qp = create_gsi_ud_qp(gsi);
298 if (IS_ERR(ptr: qp)) {
299 mlx5_ib_warn(dev,
300 "unable to create hardware UD QP for GSI: %pe\n",
301 qp);
302 return;
303 }
304
305 mqp = to_mqp(ibqp: qp);
306 if (dev->lag_active)
307 mqp->gsi_lag_port = qp_index + 1;
308 ret = modify_to_rts(gsi, qp, pkey_index);
309 if (ret)
310 goto err_destroy_qp;
311
312 spin_lock_irqsave(&gsi->lock, flags);
313 WARN_ON_ONCE(gsi->tx_qps[qp_index]);
314 gsi->tx_qps[qp_index] = qp;
315 spin_unlock_irqrestore(lock: &gsi->lock, flags);
316
317 return;
318
319err_destroy_qp:
320 WARN_ON_ONCE(qp);
321}
322
323int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
324 int attr_mask)
325{
326 struct mlx5_ib_dev *dev = to_mdev(ibdev: qp->device);
327 struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
328 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
329 u16 qp_index;
330 int ret;
331
332 mlx5_ib_dbg(dev, "modifying GSI QP to state %d\n", attr->qp_state);
333
334 ret = ib_modify_qp(qp: gsi->rx_qp, qp_attr: attr, qp_attr_mask: attr_mask);
335 if (ret) {
336 mlx5_ib_warn(dev, "unable to modify GSI rx QP: %d\n", ret);
337 return ret;
338 }
339
340 if (to_mqp(ibqp: gsi->rx_qp)->state != IB_QPS_RTS)
341 return 0;
342
343 for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index)
344 setup_qp(gsi, qp_index);
345 return 0;
346}
347
348int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
349 int qp_attr_mask,
350 struct ib_qp_init_attr *qp_init_attr)
351{
352 struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
353 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
354 int ret;
355
356 ret = ib_query_qp(qp: gsi->rx_qp, qp_attr, qp_attr_mask, qp_init_attr);
357 qp_init_attr->cap = gsi->cap;
358 return ret;
359}
360
361/* Call with gsi->lock locked */
362static int mlx5_ib_add_outstanding_wr(struct mlx5_ib_qp *mqp,
363 struct ib_ud_wr *wr, struct ib_wc *wc)
364{
365 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
366 struct mlx5_ib_dev *dev = to_mdev(ibdev: gsi->rx_qp->device);
367 struct mlx5_ib_gsi_wr *gsi_wr;
368
369 if (gsi->outstanding_pi == gsi->outstanding_ci + gsi->cap.max_send_wr) {
370 mlx5_ib_warn(dev, "no available GSI work request.\n");
371 return -ENOMEM;
372 }
373
374 gsi_wr = &gsi->outstanding_wrs[gsi->outstanding_pi %
375 gsi->cap.max_send_wr];
376 gsi->outstanding_pi++;
377
378 if (!wc) {
379 memset(&gsi_wr->wc, 0, sizeof(gsi_wr->wc));
380 gsi_wr->wc.pkey_index = wr->pkey_index;
381 gsi_wr->wc.wr_id = wr->wr.wr_id;
382 } else {
383 gsi_wr->wc = *wc;
384 gsi_wr->completed = true;
385 }
386
387 gsi_wr->cqe.done = &handle_single_completion;
388 wr->wr.wr_cqe = &gsi_wr->cqe;
389
390 return 0;
391}
392
393/* Call with gsi->lock locked */
394static int mlx5_ib_gsi_silent_drop(struct mlx5_ib_qp *mqp, struct ib_ud_wr *wr)
395{
396 struct ib_wc wc = {
397 { .wr_id = wr->wr.wr_id },
398 .status = IB_WC_SUCCESS,
399 .opcode = IB_WC_SEND,
400 .qp = &mqp->ibqp,
401 };
402 int ret;
403
404 ret = mlx5_ib_add_outstanding_wr(mqp, wr, wc: &wc);
405 if (ret)
406 return ret;
407
408 generate_completions(mqp);
409
410 return 0;
411}
412
413/* Call with gsi->lock locked */
414static struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr)
415{
416 struct mlx5_ib_dev *dev = to_mdev(ibdev: gsi->rx_qp->device);
417 struct mlx5_ib_ah *ah = to_mah(ibah: wr->ah);
418 int qp_index = wr->pkey_index;
419
420 if (!gsi->num_qps)
421 return gsi->rx_qp;
422
423 if (dev->lag_active && ah->xmit_port)
424 qp_index = ah->xmit_port - 1;
425
426 if (qp_index >= gsi->num_qps)
427 return NULL;
428
429 return gsi->tx_qps[qp_index];
430}
431
432int mlx5_ib_gsi_post_send(struct ib_qp *qp, const struct ib_send_wr *wr,
433 const struct ib_send_wr **bad_wr)
434{
435 struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
436 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
437 struct ib_qp *tx_qp;
438 unsigned long flags;
439 int ret;
440
441 for (; wr; wr = wr->next) {
442 struct ib_ud_wr cur_wr = *ud_wr(wr);
443
444 cur_wr.wr.next = NULL;
445
446 spin_lock_irqsave(&gsi->lock, flags);
447 tx_qp = get_tx_qp(gsi, wr: &cur_wr);
448 if (!tx_qp) {
449 ret = mlx5_ib_gsi_silent_drop(mqp, wr: &cur_wr);
450 if (ret)
451 goto err;
452 spin_unlock_irqrestore(lock: &gsi->lock, flags);
453 continue;
454 }
455
456 ret = mlx5_ib_add_outstanding_wr(mqp, wr: &cur_wr, NULL);
457 if (ret)
458 goto err;
459
460 ret = ib_post_send(qp: tx_qp, send_wr: &cur_wr.wr, bad_send_wr: bad_wr);
461 if (ret) {
462 /* Undo the effect of adding the outstanding wr */
463 gsi->outstanding_pi--;
464 goto err;
465 }
466 spin_unlock_irqrestore(lock: &gsi->lock, flags);
467 }
468
469 return 0;
470
471err:
472 spin_unlock_irqrestore(lock: &gsi->lock, flags);
473 *bad_wr = wr;
474 return ret;
475}
476
477int mlx5_ib_gsi_post_recv(struct ib_qp *qp, const struct ib_recv_wr *wr,
478 const struct ib_recv_wr **bad_wr)
479{
480 struct mlx5_ib_qp *mqp = to_mqp(ibqp: qp);
481 struct mlx5_ib_gsi_qp *gsi = &mqp->gsi;
482
483 return ib_post_recv(qp: gsi->rx_qp, recv_wr: wr, bad_recv_wr: bad_wr);
484}
485
486void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi)
487{
488 u16 qp_index;
489
490 for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index)
491 setup_qp(gsi, qp_index);
492}
493

source code of linux/drivers/infiniband/hw/mlx5/gsi.c