1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ |
3 | |
4 | #include "rqt.h" |
5 | #include <linux/mlx5/transobj.h> |
6 | |
7 | static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, |
8 | unsigned int size) |
9 | { |
10 | unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id); |
11 | int i; |
12 | |
13 | /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */ |
14 | for (i = 0; i < size; i++) |
15 | if (vhca_ids[i] >= max_num_vhca_id) |
16 | return false; |
17 | return true; |
18 | } |
19 | |
20 | static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, |
21 | unsigned int size) |
22 | { |
23 | if (!vhca_ids) |
24 | return true; |
25 | |
26 | if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt)) |
27 | return false; |
28 | if (!verify_num_vhca_ids(mdev, vhca_ids, size)) |
29 | return false; |
30 | |
31 | return true; |
32 | } |
33 | |
34 | void (struct mlx5e_rss_params_indir *indir, |
35 | unsigned int num_channels) |
36 | { |
37 | unsigned int i; |
38 | |
39 | for (i = 0; i < indir->actual_table_size; i++) |
40 | indir->table[i] = i % num_channels; |
41 | } |
42 | |
43 | static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size) |
44 | { |
45 | unsigned int i; |
46 | |
47 | if (vhca_ids) { |
48 | MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1); |
49 | for (i = 0; i < size; i++) { |
50 | MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]); |
51 | MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]); |
52 | } |
53 | } else { |
54 | for (i = 0; i < size; i++) |
55 | MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); |
56 | } |
57 | } |
58 | static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, |
59 | u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size) |
60 | { |
61 | int entry_sz; |
62 | void *rqtc; |
63 | int inlen; |
64 | int err; |
65 | u32 *in; |
66 | |
67 | if (!rqt_verify_vhca_ids(mdev, vhca_ids: init_vhca_ids, size: init_size)) |
68 | return -EOPNOTSUPP; |
69 | |
70 | rqt->mdev = mdev; |
71 | rqt->size = max_size; |
72 | |
73 | entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); |
74 | inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size; |
75 | in = kvzalloc(size: inlen, GFP_KERNEL); |
76 | if (!in) |
77 | return -ENOMEM; |
78 | |
79 | rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); |
80 | |
81 | MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size); |
82 | MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size); |
83 | |
84 | fill_rqn_list(rqtc, rqns: init_rqns, vhca_ids: init_vhca_ids, size: init_size); |
85 | |
86 | err = mlx5_core_create_rqt(dev: rqt->mdev, in, inlen, rqtn: &rqt->rqtn); |
87 | |
88 | kvfree(addr: in); |
89 | return err; |
90 | } |
91 | |
92 | int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, |
93 | bool indir_enabled, u32 init_rqn, u32 indir_table_size) |
94 | { |
95 | u16 max_size = indir_enabled ? indir_table_size : 1; |
96 | |
97 | return mlx5e_rqt_init(rqt, mdev, max_size, init_rqns: &init_rqn, NULL, init_size: 1); |
98 | } |
99 | |
100 | static int mlx5e_bits_invert(unsigned long a, int size) |
101 | { |
102 | int inv = 0; |
103 | int i; |
104 | |
105 | for (i = 0; i < size; i++) |
106 | inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i; |
107 | |
108 | return inv; |
109 | } |
110 | |
111 | static int mlx5e_calc_indir_rqns(u32 *, u32 *rqns, u32 *, u32 *vhca_ids, |
112 | unsigned int num_rqns, |
113 | u8 hfunc, struct mlx5e_rss_params_indir *indir) |
114 | { |
115 | unsigned int i; |
116 | |
117 | for (i = 0; i < indir->actual_table_size; i++) { |
118 | unsigned int ix = i; |
119 | |
120 | if (hfunc == ETH_RSS_HASH_XOR) |
121 | ix = mlx5e_bits_invert(a: ix, ilog2(indir->actual_table_size)); |
122 | |
123 | ix = indir->table[ix]; |
124 | |
125 | if (WARN_ON(ix >= num_rqns)) |
126 | /* Could be a bug in the driver or in the kernel part of |
127 | * ethtool: indir table refers to non-existent RQs. |
128 | */ |
129 | return -EINVAL; |
130 | rss_rqns[i] = rqns[ix]; |
131 | if (vhca_ids) |
132 | rss_vhca_ids[i] = vhca_ids[ix]; |
133 | } |
134 | |
135 | return 0; |
136 | } |
137 | |
138 | int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, |
139 | u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, |
140 | u8 hfunc, struct mlx5e_rss_params_indir *indir) |
141 | { |
142 | u32 *, * = NULL; |
143 | int err; |
144 | |
145 | rss_rqns = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_rqns), GFP_KERNEL); |
146 | if (!rss_rqns) |
147 | return -ENOMEM; |
148 | |
149 | if (vhca_ids) { |
150 | rss_vhca_ids = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_vhca_ids), |
151 | GFP_KERNEL); |
152 | if (!rss_vhca_ids) { |
153 | kvfree(addr: rss_rqns); |
154 | return -ENOMEM; |
155 | } |
156 | } |
157 | |
158 | err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); |
159 | if (err) |
160 | goto out; |
161 | |
162 | err = mlx5e_rqt_init(rqt, mdev, max_size: indir->max_table_size, init_rqns: rss_rqns, init_vhca_ids: rss_vhca_ids, |
163 | init_size: indir->actual_table_size); |
164 | |
165 | out: |
166 | kvfree(addr: rss_vhca_ids); |
167 | kvfree(addr: rss_rqns); |
168 | return err; |
169 | } |
170 | |
171 | #define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2 |
172 | |
173 | u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels) |
174 | { |
175 | u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE, |
176 | roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR)); |
177 | u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size); |
178 | |
179 | return min_t(u32, rqt_size, max_cap_rqt_size); |
180 | } |
181 | |
182 | #define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256 |
183 | |
184 | unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void) |
185 | { |
186 | return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR; |
187 | } |
188 | |
189 | void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt) |
190 | { |
191 | mlx5_core_destroy_rqt(dev: rqt->mdev, rqtn: rqt->rqtn); |
192 | } |
193 | |
194 | static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, |
195 | unsigned int size) |
196 | { |
197 | int entry_sz; |
198 | void *rqtc; |
199 | int inlen; |
200 | u32 *in; |
201 | int err; |
202 | |
203 | if (!rqt_verify_vhca_ids(mdev: rqt->mdev, vhca_ids, size)) |
204 | return -EINVAL; |
205 | |
206 | entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); |
207 | inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size; |
208 | in = kvzalloc(size: inlen, GFP_KERNEL); |
209 | if (!in) |
210 | return -ENOMEM; |
211 | |
212 | rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); |
213 | |
214 | MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); |
215 | MLX5_SET(rqtc, rqtc, rqt_actual_size, size); |
216 | |
217 | fill_rqn_list(rqtc, rqns, vhca_ids, size); |
218 | |
219 | err = mlx5_core_modify_rqt(dev: rqt->mdev, rqtn: rqt->rqtn, in, inlen); |
220 | |
221 | kvfree(addr: in); |
222 | return err; |
223 | } |
224 | |
225 | int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id) |
226 | { |
227 | return mlx5e_rqt_redirect(rqt, rqns: &rqn, vhca_ids: vhca_id, size: 1); |
228 | } |
229 | |
230 | int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, |
231 | unsigned int num_rqns, |
232 | u8 hfunc, struct mlx5e_rss_params_indir *indir) |
233 | { |
234 | u32 *, * = NULL; |
235 | int err; |
236 | |
237 | if (!rqt_verify_vhca_ids(mdev: rqt->mdev, vhca_ids, size: num_rqns)) |
238 | return -EINVAL; |
239 | |
240 | if (WARN_ON(rqt->size != indir->max_table_size)) |
241 | return -EINVAL; |
242 | |
243 | rss_rqns = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_rqns), GFP_KERNEL); |
244 | if (!rss_rqns) |
245 | return -ENOMEM; |
246 | |
247 | if (vhca_ids) { |
248 | rss_vhca_ids = kvmalloc_array(n: indir->actual_table_size, size: sizeof(*rss_vhca_ids), |
249 | GFP_KERNEL); |
250 | if (!rss_vhca_ids) { |
251 | kvfree(addr: rss_rqns); |
252 | return -ENOMEM; |
253 | } |
254 | } |
255 | |
256 | err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); |
257 | if (err) |
258 | goto out; |
259 | |
260 | err = mlx5e_rqt_redirect(rqt, rqns: rss_rqns, vhca_ids: rss_vhca_ids, size: indir->actual_table_size); |
261 | |
262 | out: |
263 | kvfree(addr: rss_vhca_ids); |
264 | kvfree(addr: rss_rqns); |
265 | return err; |
266 | } |
267 | |