1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3
4#include "rqt.h"
5#include <linux/mlx5/transobj.h>
6
7static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids,
8 unsigned int size)
9{
10 unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id);
11 int i;
12
13 /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */
14 for (i = 0; i < size; i++)
15 if (vhca_ids[i] >= max_num_vhca_id)
16 return false;
17 return true;
18}
19
20static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids,
21 unsigned int size)
22{
23 if (!vhca_ids)
24 return true;
25
26 if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt))
27 return false;
28 if (!verify_num_vhca_ids(mdev, vhca_ids, size))
29 return false;
30
31 return true;
32}
33
34void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir,
35 unsigned int num_channels)
36{
37 unsigned int i;
38
39 for (i = 0; i < indir->actual_table_size; i++)
40 indir->table[i] = i % num_channels;
41}
42
43static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size)
44{
45 unsigned int i;
46
47 if (vhca_ids) {
48 MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1);
49 for (i = 0; i < size; i++) {
50 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]);
51 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]);
52 }
53 } else {
54 for (i = 0; i < size; i++)
55 MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]);
56 }
57}
58static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
59 u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size)
60{
61 int entry_sz;
62 void *rqtc;
63 int inlen;
64 int err;
65 u32 *in;
66
67 if (!rqt_verify_vhca_ids(mdev, vhca_ids: init_vhca_ids, size: init_size))
68 return -EOPNOTSUPP;
69
70 rqt->mdev = mdev;
71 rqt->size = max_size;
72
73 entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num);
74 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size;
75 in = kvzalloc(size: inlen, GFP_KERNEL);
76 if (!in)
77 return -ENOMEM;
78
79 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
80
81 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size);
82 MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size);
83
84 fill_rqn_list(rqtc, rqns: init_rqns, vhca_ids: init_vhca_ids, size: init_size);
85
86 err = mlx5_core_create_rqt(dev: rqt->mdev, in, inlen, rqtn: &rqt->rqtn);
87
88 kvfree(addr: in);
89 return err;
90}
91
92int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
93 bool indir_enabled, u32 init_rqn, u32 indir_table_size)
94{
95 u16 max_size = indir_enabled ? indir_table_size : 1;
96
97 return mlx5e_rqt_init(rqt, mdev, max_size, init_rqns: &init_rqn, NULL, init_size: 1);
98}
99
100static int mlx5e_bits_invert(unsigned long a, int size)
101{
102 int inv = 0;
103 int i;
104
105 for (i = 0; i < size; i++)
106 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
107
108 return inv;
109}
110
111static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids,
112 unsigned int num_rqns,
113 u8 hfunc, struct mlx5e_rss_params_indir *indir)
114{
115 unsigned int i;
116
117 for (i = 0; i < indir->actual_table_size; i++) {
118 unsigned int ix = i;
119
120 if (hfunc == ETH_RSS_HASH_XOR)
121 ix = mlx5e_bits_invert(a: ix, ilog2(indir->actual_table_size));
122
123 ix = indir->table[ix];
124
125 if (WARN_ON(ix >= num_rqns))
126 /* Could be a bug in the driver or in the kernel part of
127 * ethtool: indir table refers to non-existent RQs.
128 */
129 return -EINVAL;
130 rss_rqns[i] = rqns[ix];
131 if (vhca_ids)
132 rss_vhca_ids[i] = vhca_ids[ix];
133 }
134
135 return 0;
136}
137
138int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
139 u32 *rqns, u32 *vhca_ids, unsigned int num_rqns,
140 u8 hfunc, struct mlx5e_rss_params_indir *indir)
141{
142 u32 *rss_rqns, *rss_vhca_ids = NULL;
143 int err;
144
145 rss_rqns = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_rqns), GFP_KERNEL);
146 if (!rss_rqns)
147 return -ENOMEM;
148
149 if (vhca_ids) {
150 rss_vhca_ids = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_vhca_ids),
151 GFP_KERNEL);
152 if (!rss_vhca_ids) {
153 kvfree(addr: rss_rqns);
154 return -ENOMEM;
155 }
156 }
157
158 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir);
159 if (err)
160 goto out;
161
162 err = mlx5e_rqt_init(rqt, mdev, max_size: indir->max_table_size, init_rqns: rss_rqns, init_vhca_ids: rss_vhca_ids,
163 init_size: indir->actual_table_size);
164
165out:
166 kvfree(addr: rss_vhca_ids);
167 kvfree(addr: rss_rqns);
168 return err;
169}
170
171#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2
172
173u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)
174{
175 u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE,
176 roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR));
177 u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size);
178
179 return min_t(u32, rqt_size, max_cap_rqt_size);
180}
181
182#define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256
183
184unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void)
185{
186 return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR;
187}
188
189void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt)
190{
191 mlx5_core_destroy_rqt(dev: rqt->mdev, rqtn: rqt->rqtn);
192}
193
194static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
195 unsigned int size)
196{
197 int entry_sz;
198 void *rqtc;
199 int inlen;
200 u32 *in;
201 int err;
202
203 if (!rqt_verify_vhca_ids(mdev: rqt->mdev, vhca_ids, size))
204 return -EINVAL;
205
206 entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num);
207 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size;
208 in = kvzalloc(size: inlen, GFP_KERNEL);
209 if (!in)
210 return -ENOMEM;
211
212 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
213
214 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
215 MLX5_SET(rqtc, rqtc, rqt_actual_size, size);
216
217 fill_rqn_list(rqtc, rqns, vhca_ids, size);
218
219 err = mlx5_core_modify_rqt(dev: rqt->mdev, rqtn: rqt->rqtn, in, inlen);
220
221 kvfree(addr: in);
222 return err;
223}
224
225int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id)
226{
227 return mlx5e_rqt_redirect(rqt, rqns: &rqn, vhca_ids: vhca_id, size: 1);
228}
229
230int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
231 unsigned int num_rqns,
232 u8 hfunc, struct mlx5e_rss_params_indir *indir)
233{
234 u32 *rss_rqns, *rss_vhca_ids = NULL;
235 int err;
236
237 if (!rqt_verify_vhca_ids(mdev: rqt->mdev, vhca_ids, size: num_rqns))
238 return -EINVAL;
239
240 if (WARN_ON(rqt->size != indir->max_table_size))
241 return -EINVAL;
242
243 rss_rqns = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_rqns), GFP_KERNEL);
244 if (!rss_rqns)
245 return -ENOMEM;
246
247 if (vhca_ids) {
248 rss_vhca_ids = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_vhca_ids),
249 GFP_KERNEL);
250 if (!rss_vhca_ids) {
251 kvfree(addr: rss_rqns);
252 return -ENOMEM;
253 }
254 }
255
256 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir);
257 if (err)
258 goto out;
259
260 err = mlx5e_rqt_redirect(rqt, rqns: rss_rqns, vhca_ids: rss_vhca_ids, size: indir->actual_table_size);
261
262out:
263 kvfree(addr: rss_vhca_ids);
264 kvfree(addr: rss_rqns);
265 return err;
266}
267

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c