1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/mlx5/driver.h>
34#include "mlx5_core.h"
35#include <linux/mlx5/transobj.h>
36
37int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
38{
39 u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
40 u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
41 int err;
42
43 MLX5_SET(alloc_transport_domain_in, in, opcode,
44 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
45
46 err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);
47 if (!err)
48 *tdn = MLX5_GET(alloc_transport_domain_out, out,
49 transport_domain);
50
51 return err;
52}
53EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
54
55void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
56{
57 u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};
58
59 MLX5_SET(dealloc_transport_domain_in, in, opcode,
60 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
61 MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
62 mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);
63}
64EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
65
66int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
67{
68 u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {};
69 int err;
70
71 MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
72 err = mlx5_cmd_exec(dev, in, in_size: inlen, out, out_size: sizeof(out));
73 if (!err)
74 *rqn = MLX5_GET(create_rq_out, out, rqn);
75
76 return err;
77}
78EXPORT_SYMBOL(mlx5_core_create_rq);
79
80int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in)
81{
82 MLX5_SET(modify_rq_in, in, rqn, rqn);
83 MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
84
85 return mlx5_cmd_exec_in(dev, modify_rq, in);
86}
87EXPORT_SYMBOL(mlx5_core_modify_rq);
88
89void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
90{
91 u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};
92
93 MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
94 MLX5_SET(destroy_rq_in, in, rqn, rqn);
95 mlx5_cmd_exec_in(dev, destroy_rq, in);
96}
97EXPORT_SYMBOL(mlx5_core_destroy_rq);
98
99int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
100{
101 u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {};
102
103 MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
104 MLX5_SET(query_rq_in, in, rqn, rqn);
105
106 return mlx5_cmd_exec_inout(dev, query_rq, in, out);
107}
108EXPORT_SYMBOL(mlx5_core_query_rq);
109
110int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
111{
112 u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};
113 int err;
114
115 MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
116 err = mlx5_cmd_exec(dev, in, in_size: inlen, out, out_size: sizeof(out));
117 if (!err)
118 *sqn = MLX5_GET(create_sq_out, out, sqn);
119
120 return err;
121}
122
123int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in)
124{
125 MLX5_SET(modify_sq_in, in, sqn, sqn);
126 MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
127 return mlx5_cmd_exec_in(dev, modify_sq, in);
128}
129EXPORT_SYMBOL(mlx5_core_modify_sq);
130
131void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
132{
133 u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};
134
135 MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
136 MLX5_SET(destroy_sq_in, in, sqn, sqn);
137 mlx5_cmd_exec_in(dev, destroy_sq, in);
138}
139
140int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
141{
142 u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {};
143
144 MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
145 MLX5_SET(query_sq_in, in, sqn, sqn);
146 return mlx5_cmd_exec_inout(dev, query_sq, in, out);
147}
148EXPORT_SYMBOL(mlx5_core_query_sq);
149
150int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state)
151{
152 void *out;
153 void *sqc;
154 int inlen;
155 int err;
156
157 inlen = MLX5_ST_SZ_BYTES(query_sq_out);
158 out = kvzalloc(size: inlen, GFP_KERNEL);
159 if (!out)
160 return -ENOMEM;
161
162 err = mlx5_core_query_sq(dev, sqn, out);
163 if (err)
164 goto out;
165
166 sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
167 *state = MLX5_GET(sqc, sqc, state);
168
169out:
170 kvfree(addr: out);
171 return err;
172}
173EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);
174
175int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, u32 *tirn)
176{
177 u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
178 int err;
179
180 MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
181 err = mlx5_cmd_exec_inout(dev, create_tir, in, out);
182 if (!err)
183 *tirn = MLX5_GET(create_tir_out, out, tirn);
184
185 return err;
186}
187EXPORT_SYMBOL(mlx5_core_create_tir);
188
189int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in)
190{
191 MLX5_SET(modify_tir_in, in, tirn, tirn);
192 MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
193 return mlx5_cmd_exec_in(dev, modify_tir, in);
194}
195
196void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
197{
198 u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};
199
200 MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
201 MLX5_SET(destroy_tir_in, in, tirn, tirn);
202 mlx5_cmd_exec_in(dev, destroy_tir, in);
203}
204EXPORT_SYMBOL(mlx5_core_destroy_tir);
205
206int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, u32 *tisn)
207{
208 u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
209 int err;
210
211 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
212 err = mlx5_cmd_exec_inout(dev, create_tis, in, out);
213 if (!err)
214 *tisn = MLX5_GET(create_tis_out, out, tisn);
215
216 return err;
217}
218EXPORT_SYMBOL(mlx5_core_create_tis);
219
220int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in)
221{
222 MLX5_SET(modify_tis_in, in, tisn, tisn);
223 MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
224
225 return mlx5_cmd_exec_in(dev, modify_tis, in);
226}
227EXPORT_SYMBOL(mlx5_core_modify_tis);
228
229void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
230{
231 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};
232
233 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
234 MLX5_SET(destroy_tis_in, in, tisn, tisn);
235 mlx5_cmd_exec_in(dev, destroy_tis, in);
236}
237EXPORT_SYMBOL(mlx5_core_destroy_tis);
238
239int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
240 u32 *rqtn)
241{
242 u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
243 int err;
244
245 MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
246 err = mlx5_cmd_exec(dev, in, in_size: inlen, out, out_size: sizeof(out));
247 if (!err)
248 *rqtn = MLX5_GET(create_rqt_out, out, rqtn);
249
250 return err;
251}
252EXPORT_SYMBOL(mlx5_core_create_rqt);
253
254int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
255 int inlen)
256{
257 u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {};
258
259 MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
260 MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
261 return mlx5_cmd_exec(dev, in, in_size: inlen, out, out_size: sizeof(out));
262}
263
264void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
265{
266 u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};
267
268 MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
269 MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
270 mlx5_cmd_exec_in(dev, destroy_rqt, in);
271}
272EXPORT_SYMBOL(mlx5_core_destroy_rqt);
273
274static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
275 struct mlx5_hairpin_params *params, u32 *rqn)
276{
277 u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
278 void *rqc, *wq;
279
280 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
281 wq = MLX5_ADDR_OF(rqc, rqc, wq);
282
283 MLX5_SET(rqc, rqc, hairpin, 1);
284 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
285 MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
286
287 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
288 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
289
290 return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
291}
292
293static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
294 struct mlx5_hairpin_params *params, u32 *sqn)
295{
296 u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
297 void *sqc, *wq;
298
299 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
300 wq = MLX5_ADDR_OF(sqc, sqc, wq);
301
302 MLX5_SET(sqc, sqc, hairpin, 1);
303 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
304
305 MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
306 MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
307
308 return mlx5_core_create_sq(dev: mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
309}
310
311static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
312 struct mlx5_hairpin_params *params)
313{
314 int i, j, err;
315
316 for (i = 0; i < hp->num_channels; i++) {
317 err = mlx5_hairpin_create_rq(mdev: hp->func_mdev, params, rqn: &hp->rqn[i]);
318 if (err)
319 goto out_err_rq;
320 }
321
322 for (i = 0; i < hp->num_channels; i++) {
323 err = mlx5_hairpin_create_sq(mdev: hp->peer_mdev, params, sqn: &hp->sqn[i]);
324 if (err)
325 goto out_err_sq;
326 }
327
328 return 0;
329
330out_err_sq:
331 for (j = 0; j < i; j++)
332 mlx5_core_destroy_sq(dev: hp->peer_mdev, sqn: hp->sqn[j]);
333 i = hp->num_channels;
334out_err_rq:
335 for (j = 0; j < i; j++)
336 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
337 return err;
338}
339
340static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
341{
342 int i;
343
344 for (i = 0; i < hp->num_channels; i++) {
345 mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
346 if (!hp->peer_gone)
347 mlx5_core_destroy_sq(dev: hp->peer_mdev, sqn: hp->sqn[i]);
348 }
349}
350
351static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
352 int curr_state, int next_state,
353 u16 peer_vhca, u32 peer_sq)
354{
355 u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
356 void *rqc;
357
358 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
359
360 if (next_state == MLX5_RQC_STATE_RDY) {
361 MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
362 MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
363 }
364
365 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
366 MLX5_SET(rqc, rqc, state, next_state);
367
368 return mlx5_core_modify_rq(func_mdev, rqn, in);
369}
370
371static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
372 int curr_state, int next_state,
373 u16 peer_vhca, u32 peer_rq)
374{
375 u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
376 void *sqc;
377
378 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
379
380 if (next_state == MLX5_SQC_STATE_RDY) {
381 MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
382 MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
383 }
384
385 MLX5_SET(modify_sq_in, in, sq_state, curr_state);
386 MLX5_SET(sqc, sqc, state, next_state);
387
388 return mlx5_core_modify_sq(peer_mdev, sqn, in);
389}
390
391static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
392{
393 int i, j, err;
394
395 /* set peer SQs */
396 for (i = 0; i < hp->num_channels; i++) {
397 err = mlx5_hairpin_modify_sq(peer_mdev: hp->peer_mdev, sqn: hp->sqn[i],
398 curr_state: MLX5_SQC_STATE_RST, next_state: MLX5_SQC_STATE_RDY,
399 MLX5_CAP_GEN(hp->func_mdev, vhca_id), peer_rq: hp->rqn[i]);
400 if (err)
401 goto err_modify_sq;
402 }
403
404 /* set func RQs */
405 for (i = 0; i < hp->num_channels; i++) {
406 err = mlx5_hairpin_modify_rq(func_mdev: hp->func_mdev, rqn: hp->rqn[i],
407 curr_state: MLX5_RQC_STATE_RST, next_state: MLX5_RQC_STATE_RDY,
408 MLX5_CAP_GEN(hp->peer_mdev, vhca_id), peer_sq: hp->sqn[i]);
409 if (err)
410 goto err_modify_rq;
411 }
412
413 return 0;
414
415err_modify_rq:
416 for (j = 0; j < i; j++)
417 mlx5_hairpin_modify_rq(func_mdev: hp->func_mdev, rqn: hp->rqn[j], curr_state: MLX5_RQC_STATE_RDY,
418 next_state: MLX5_RQC_STATE_RST, peer_vhca: 0, peer_sq: 0);
419 i = hp->num_channels;
420err_modify_sq:
421 for (j = 0; j < i; j++)
422 mlx5_hairpin_modify_sq(peer_mdev: hp->peer_mdev, sqn: hp->sqn[j], curr_state: MLX5_SQC_STATE_RDY,
423 next_state: MLX5_SQC_STATE_RST, peer_vhca: 0, peer_rq: 0);
424 return err;
425}
426
427static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp)
428{
429 int i;
430
431 for (i = 0; i < hp->num_channels; i++)
432 mlx5_hairpin_modify_sq(peer_mdev: hp->peer_mdev, sqn: hp->sqn[i], curr_state: MLX5_SQC_STATE_RDY,
433 next_state: MLX5_SQC_STATE_RST, peer_vhca: 0, peer_rq: 0);
434}
435
436static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
437{
438 int i;
439
440 /* unset func RQs */
441 for (i = 0; i < hp->num_channels; i++)
442 mlx5_hairpin_modify_rq(func_mdev: hp->func_mdev, rqn: hp->rqn[i], curr_state: MLX5_RQC_STATE_RDY,
443 next_state: MLX5_RQC_STATE_RST, peer_vhca: 0, peer_sq: 0);
444 /* unset peer SQs */
445 if (!hp->peer_gone)
446 mlx5_hairpin_unpair_peer_sq(hp);
447}
448
449struct mlx5_hairpin *
450mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
451 struct mlx5_core_dev *peer_mdev,
452 struct mlx5_hairpin_params *params)
453{
454 struct mlx5_hairpin *hp;
455 int size, err;
456
457 size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
458 hp = kzalloc(size, GFP_KERNEL);
459 if (!hp)
460 return ERR_PTR(error: -ENOMEM);
461
462 hp->func_mdev = func_mdev;
463 hp->peer_mdev = peer_mdev;
464 hp->num_channels = params->num_channels;
465
466 hp->rqn = (void *)hp + sizeof(*hp);
467 hp->sqn = hp->rqn + params->num_channels;
468
469 /* alloc and pair func --> peer hairpin */
470 err = mlx5_hairpin_create_queues(hp, params);
471 if (err)
472 goto err_create_queues;
473
474 err = mlx5_hairpin_pair_queues(hp);
475 if (err)
476 goto err_pair_queues;
477
478 return hp;
479
480err_pair_queues:
481 mlx5_hairpin_destroy_queues(hp);
482err_create_queues:
483 kfree(objp: hp);
484 return ERR_PTR(error: err);
485}
486
487void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
488{
489 mlx5_hairpin_unpair_queues(hp);
490 mlx5_hairpin_destroy_queues(hp);
491 kfree(objp: hp);
492}
493
494void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp)
495{
496 int i;
497
498 mlx5_hairpin_unpair_peer_sq(hp);
499
500 /* destroy peer SQ */
501 for (i = 0; i < hp->num_channels; i++)
502 mlx5_core_destroy_sq(dev: hp->peer_mdev, sqn: hp->sqn[i]);
503
504 hp->peer_gone = true;
505}
506

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/transobj.c