1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. |
3 | |
4 | #include "rss.h" |
5 | |
6 | #define (__dev, format, ...) \ |
7 | dev_warn((__dev)->device, "%s:%d:(pid %d): " format, \ |
8 | __func__, __LINE__, current->pid, \ |
9 | ##__VA_ARGS__) |
10 | |
11 | static const struct mlx5e_rss_params_traffic_type [MLX5E_NUM_INDIR_TIRS] = { |
12 | [MLX5_TT_IPV4_TCP] = { |
13 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, |
14 | .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, |
15 | .rx_hash_fields = MLX5_HASH_IP_L4PORTS, |
16 | }, |
17 | [MLX5_TT_IPV6_TCP] = { |
18 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, |
19 | .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, |
20 | .rx_hash_fields = MLX5_HASH_IP_L4PORTS, |
21 | }, |
22 | [MLX5_TT_IPV4_UDP] = { |
23 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, |
24 | .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, |
25 | .rx_hash_fields = MLX5_HASH_IP_L4PORTS, |
26 | }, |
27 | [MLX5_TT_IPV6_UDP] = { |
28 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, |
29 | .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, |
30 | .rx_hash_fields = MLX5_HASH_IP_L4PORTS, |
31 | }, |
32 | [MLX5_TT_IPV4_IPSEC_AH] = { |
33 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, |
34 | .l4_prot_type = 0, |
35 | .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, |
36 | }, |
37 | [MLX5_TT_IPV6_IPSEC_AH] = { |
38 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, |
39 | .l4_prot_type = 0, |
40 | .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, |
41 | }, |
42 | [MLX5_TT_IPV4_IPSEC_ESP] = { |
43 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, |
44 | .l4_prot_type = 0, |
45 | .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, |
46 | }, |
47 | [MLX5_TT_IPV6_IPSEC_ESP] = { |
48 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, |
49 | .l4_prot_type = 0, |
50 | .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, |
51 | }, |
52 | [MLX5_TT_IPV4] = { |
53 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, |
54 | .l4_prot_type = 0, |
55 | .rx_hash_fields = MLX5_HASH_IP, |
56 | }, |
57 | [MLX5_TT_IPV6] = { |
58 | .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, |
59 | .l4_prot_type = 0, |
60 | .rx_hash_fields = MLX5_HASH_IP, |
61 | }, |
62 | }; |
63 | |
64 | struct mlx5e_rss_params_traffic_type |
65 | (enum mlx5_traffic_types tt) |
66 | { |
67 | return rss_default_config[tt]; |
68 | } |
69 | |
70 | struct { |
71 | struct mlx5e_rss_params_hash ; |
72 | struct mlx5e_rss_params_indir ; |
73 | u32 [MLX5E_NUM_INDIR_TIRS]; |
74 | struct mlx5e_tir *[MLX5E_NUM_INDIR_TIRS]; |
75 | struct mlx5e_tir *[MLX5E_NUM_INDIR_TIRS]; |
76 | struct mlx5e_rqt ; |
77 | struct mlx5_core_dev *; /* primary */ |
78 | u32 ; |
79 | bool ; |
80 | bool ; |
81 | refcount_t ; |
82 | }; |
83 | |
84 | void (struct mlx5e_rss *, u32 num_channels) |
85 | { |
86 | rss->indir.actual_table_size = mlx5e_rqt_size(mdev: rss->mdev, num_channels); |
87 | } |
88 | |
89 | int (struct mlx5e_rss_params_indir *indir, struct mlx5_core_dev *mdev, |
90 | u32 actual_table_size, u32 max_table_size) |
91 | { |
92 | indir->table = kvmalloc_array(n: max_table_size, size: sizeof(*indir->table), GFP_KERNEL); |
93 | if (!indir->table) |
94 | return -ENOMEM; |
95 | |
96 | indir->max_table_size = max_table_size; |
97 | indir->actual_table_size = actual_table_size; |
98 | |
99 | return 0; |
100 | } |
101 | |
102 | void (struct mlx5e_rss_params_indir *indir) |
103 | { |
104 | kvfree(addr: indir->table); |
105 | } |
106 | |
107 | static int (struct mlx5e_rss *to, const struct mlx5e_rss *from) |
108 | { |
109 | u32 *dst_indir_table; |
110 | |
111 | if (to->indir.actual_table_size != from->indir.actual_table_size || |
112 | to->indir.max_table_size != from->indir.max_table_size) { |
113 | mlx5e_rss_warn(to->mdev, |
114 | "Failed to copy RSS due to size mismatch, src (actual %u, max %u) != dst (actual %u, max %u)\n" , |
115 | from->indir.actual_table_size, from->indir.max_table_size, |
116 | to->indir.actual_table_size, to->indir.max_table_size); |
117 | return -EINVAL; |
118 | } |
119 | |
120 | dst_indir_table = to->indir.table; |
121 | *to = *from; |
122 | to->indir.table = dst_indir_table; |
123 | memcpy(to->indir.table, from->indir.table, |
124 | from->indir.actual_table_size * sizeof(*from->indir.table)); |
125 | return 0; |
126 | } |
127 | |
128 | static struct mlx5e_rss *(const struct mlx5e_rss *from) |
129 | { |
130 | struct mlx5e_rss *; |
131 | int err; |
132 | |
133 | rss = kvzalloc(size: sizeof(*rss), GFP_KERNEL); |
134 | if (!rss) |
135 | return ERR_PTR(error: -ENOMEM); |
136 | |
137 | err = mlx5e_rss_params_indir_init(indir: &rss->indir, mdev: from->mdev, actual_table_size: from->indir.actual_table_size, |
138 | max_table_size: from->indir.max_table_size); |
139 | if (err) |
140 | goto err_free_rss; |
141 | |
142 | err = mlx5e_rss_copy(to: rss, from); |
143 | if (err) |
144 | goto err_free_indir; |
145 | |
146 | return rss; |
147 | |
148 | err_free_indir: |
149 | mlx5e_rss_params_indir_cleanup(indir: &rss->indir); |
150 | : |
151 | kvfree(addr: rss); |
152 | return ERR_PTR(error: err); |
153 | } |
154 | |
155 | static void (struct mlx5e_rss *) |
156 | { |
157 | enum mlx5_traffic_types tt; |
158 | |
159 | rss->hash.hfunc = ETH_RSS_HASH_TOP; |
160 | netdev_rss_key_fill(buffer: rss->hash.toeplitz_hash_key, |
161 | len: sizeof(rss->hash.toeplitz_hash_key)); |
162 | for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) |
163 | rss->rx_hash_fields[tt] = |
164 | mlx5e_rss_get_default_tt_config(tt).rx_hash_fields; |
165 | } |
166 | |
167 | static struct mlx5e_tir **(struct mlx5e_rss *, enum mlx5_traffic_types tt, |
168 | bool inner) |
169 | { |
170 | return inner ? &rss->inner_tir[tt] : &rss->tir[tt]; |
171 | } |
172 | |
173 | static struct mlx5e_tir *(struct mlx5e_rss *, enum mlx5_traffic_types tt, |
174 | bool inner) |
175 | { |
176 | return *rss_get_tirp(rss, tt, inner); |
177 | } |
178 | |
179 | static struct mlx5e_rss_params_traffic_type |
180 | (struct mlx5e_rss *, enum mlx5_traffic_types tt) |
181 | { |
182 | struct mlx5e_rss_params_traffic_type ; |
183 | |
184 | rss_tt = mlx5e_rss_get_default_tt_config(tt); |
185 | rss_tt.rx_hash_fields = rss->rx_hash_fields[tt]; |
186 | return rss_tt; |
187 | } |
188 | |
189 | static int (struct mlx5e_rss *, |
190 | enum mlx5_traffic_types tt, |
191 | const struct mlx5e_packet_merge_param *init_pkt_merge_param, |
192 | bool inner) |
193 | { |
194 | struct mlx5e_rss_params_traffic_type ; |
195 | struct mlx5e_tir_builder *builder; |
196 | struct mlx5e_tir **tir_p; |
197 | struct mlx5e_tir *tir; |
198 | u32 rqtn; |
199 | int err; |
200 | |
201 | if (inner && !rss->inner_ft_support) { |
202 | mlx5e_rss_warn(rss->mdev, |
203 | "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n" , |
204 | tt); |
205 | return -EINVAL; |
206 | } |
207 | |
208 | tir_p = rss_get_tirp(rss, tt, inner); |
209 | if (*tir_p) |
210 | return -EINVAL; |
211 | |
212 | tir = kvzalloc(size: sizeof(*tir), GFP_KERNEL); |
213 | if (!tir) |
214 | return -ENOMEM; |
215 | |
216 | builder = mlx5e_tir_builder_alloc(modify: false); |
217 | if (!builder) { |
218 | err = -ENOMEM; |
219 | goto free_tir; |
220 | } |
221 | |
222 | rqtn = mlx5e_rqt_get_rqtn(rqt: &rss->rqt); |
223 | mlx5e_tir_builder_build_rqt(builder, tdn: rss->mdev->mlx5e_res.hw_objs.td.tdn, |
224 | rqtn, inner_ft_support: rss->inner_ft_support); |
225 | mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param: init_pkt_merge_param); |
226 | rss_tt = mlx5e_rss_get_tt_config(rss, tt); |
227 | mlx5e_tir_builder_build_rss(builder, rss_hash: &rss->hash, rss_tt: &rss_tt, inner); |
228 | |
229 | err = mlx5e_tir_init(tir, builder, mdev: rss->mdev, reg: true); |
230 | mlx5e_tir_builder_free(builder); |
231 | if (err) { |
232 | mlx5e_rss_warn(rss->mdev, "Failed to create %sindirect TIR: err = %d, tt = %d\n" , |
233 | inner ? "inner " : "" , err, tt); |
234 | goto free_tir; |
235 | } |
236 | |
237 | *tir_p = tir; |
238 | return 0; |
239 | |
240 | free_tir: |
241 | kvfree(addr: tir); |
242 | return err; |
243 | } |
244 | |
245 | static void (struct mlx5e_rss *, enum mlx5_traffic_types tt, |
246 | bool inner) |
247 | { |
248 | struct mlx5e_tir **tir_p; |
249 | struct mlx5e_tir *tir; |
250 | |
251 | tir_p = rss_get_tirp(rss, tt, inner); |
252 | if (!*tir_p) |
253 | return; |
254 | |
255 | tir = *tir_p; |
256 | mlx5e_tir_destroy(tir); |
257 | kvfree(addr: tir); |
258 | *tir_p = NULL; |
259 | } |
260 | |
261 | static int (struct mlx5e_rss *, |
262 | const struct mlx5e_packet_merge_param *init_pkt_merge_param, |
263 | bool inner) |
264 | { |
265 | enum mlx5_traffic_types tt, max_tt; |
266 | int err; |
267 | |
268 | for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { |
269 | err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner); |
270 | if (err) |
271 | goto err_destroy_tirs; |
272 | } |
273 | |
274 | return 0; |
275 | |
276 | err_destroy_tirs: |
277 | max_tt = tt; |
278 | for (tt = 0; tt < max_tt; tt++) |
279 | mlx5e_rss_destroy_tir(rss, tt, inner); |
280 | return err; |
281 | } |
282 | |
283 | static void (struct mlx5e_rss *, bool inner) |
284 | { |
285 | enum mlx5_traffic_types tt; |
286 | |
287 | for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) |
288 | mlx5e_rss_destroy_tir(rss, tt, inner); |
289 | } |
290 | |
291 | static int (struct mlx5e_rss *, enum mlx5_traffic_types tt, |
292 | bool inner) |
293 | { |
294 | struct mlx5e_rss_params_traffic_type ; |
295 | struct mlx5e_tir_builder *builder; |
296 | struct mlx5e_tir *tir; |
297 | int err; |
298 | |
299 | tir = rss_get_tir(rss, tt, inner); |
300 | if (!tir) |
301 | return 0; |
302 | |
303 | builder = mlx5e_tir_builder_alloc(modify: true); |
304 | if (!builder) |
305 | return -ENOMEM; |
306 | |
307 | rss_tt = mlx5e_rss_get_tt_config(rss, tt); |
308 | |
309 | mlx5e_tir_builder_build_rss(builder, rss_hash: &rss->hash, rss_tt: &rss_tt, inner); |
310 | err = mlx5e_tir_modify(tir, builder); |
311 | |
312 | mlx5e_tir_builder_free(builder); |
313 | return err; |
314 | } |
315 | |
316 | static int (struct mlx5e_rss *) |
317 | { |
318 | enum mlx5_traffic_types tt; |
319 | int err, retval; |
320 | |
321 | retval = 0; |
322 | |
323 | for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { |
324 | err = mlx5e_rss_update_tir(rss, tt, inner: false); |
325 | if (err) { |
326 | retval = retval ? : err; |
327 | mlx5e_rss_warn(rss->mdev, |
328 | "Failed to update RSS hash of indirect TIR for traffic type %d: err = %d\n" , |
329 | tt, err); |
330 | } |
331 | |
332 | if (!rss->inner_ft_support) |
333 | continue; |
334 | |
335 | err = mlx5e_rss_update_tir(rss, tt, inner: true); |
336 | if (err) { |
337 | retval = retval ? : err; |
338 | mlx5e_rss_warn(rss->mdev, |
339 | "Failed to update RSS hash of inner indirect TIR for traffic type %d: err = %d\n" , |
340 | tt, err); |
341 | } |
342 | } |
343 | return retval; |
344 | } |
345 | |
346 | static int (struct mlx5e_rss *) |
347 | { |
348 | mlx5e_rss_params_init(rss); |
349 | refcount_set(r: &rss->refcnt, n: 1); |
350 | |
351 | return mlx5e_rqt_init_direct(rqt: &rss->rqt, mdev: rss->mdev, indir_enabled: true, |
352 | init_rqn: rss->drop_rqn, indir_table_size: rss->indir.max_table_size); |
353 | } |
354 | |
355 | struct mlx5e_rss *(struct mlx5_core_dev *mdev, bool inner_ft_support, u32 drop_rqn, |
356 | const struct mlx5e_packet_merge_param *init_pkt_merge_param, |
357 | enum mlx5e_rss_init_type type, unsigned int nch, |
358 | unsigned int max_nch) |
359 | { |
360 | struct mlx5e_rss *; |
361 | int err; |
362 | |
363 | rss = kvzalloc(size: sizeof(*rss), GFP_KERNEL); |
364 | if (!rss) |
365 | return ERR_PTR(error: -ENOMEM); |
366 | |
367 | err = mlx5e_rss_params_indir_init(indir: &rss->indir, mdev, |
368 | actual_table_size: mlx5e_rqt_size(mdev, num_channels: nch), |
369 | max_table_size: mlx5e_rqt_size(mdev, num_channels: max_nch)); |
370 | if (err) |
371 | goto err_free_rss; |
372 | |
373 | rss->mdev = mdev; |
374 | rss->inner_ft_support = inner_ft_support; |
375 | rss->drop_rqn = drop_rqn; |
376 | |
377 | err = mlx5e_rss_init_no_tirs(rss); |
378 | if (err) |
379 | goto err_free_indir; |
380 | |
381 | if (type == MLX5E_RSS_INIT_NO_TIRS) |
382 | goto out; |
383 | |
384 | err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, inner: false); |
385 | if (err) |
386 | goto err_destroy_rqt; |
387 | |
388 | if (inner_ft_support) { |
389 | err = mlx5e_rss_create_tirs(rss, init_pkt_merge_param, inner: true); |
390 | if (err) |
391 | goto err_destroy_tirs; |
392 | } |
393 | |
394 | out: |
395 | return rss; |
396 | |
397 | err_destroy_tirs: |
398 | mlx5e_rss_destroy_tirs(rss, inner: false); |
399 | err_destroy_rqt: |
400 | mlx5e_rqt_destroy(rqt: &rss->rqt); |
401 | err_free_indir: |
402 | mlx5e_rss_params_indir_cleanup(indir: &rss->indir); |
403 | : |
404 | kvfree(addr: rss); |
405 | return ERR_PTR(error: err); |
406 | } |
407 | |
408 | int (struct mlx5e_rss *) |
409 | { |
410 | if (!refcount_dec_if_one(r: &rss->refcnt)) |
411 | return -EBUSY; |
412 | |
413 | mlx5e_rss_destroy_tirs(rss, inner: false); |
414 | |
415 | if (rss->inner_ft_support) |
416 | mlx5e_rss_destroy_tirs(rss, inner: true); |
417 | |
418 | mlx5e_rqt_destroy(rqt: &rss->rqt); |
419 | mlx5e_rss_params_indir_cleanup(indir: &rss->indir); |
420 | kvfree(addr: rss); |
421 | |
422 | return 0; |
423 | } |
424 | |
425 | void (struct mlx5e_rss *) |
426 | { |
427 | refcount_inc(r: &rss->refcnt); |
428 | } |
429 | |
430 | void (struct mlx5e_rss *) |
431 | { |
432 | refcount_dec(r: &rss->refcnt); |
433 | } |
434 | |
435 | unsigned int (struct mlx5e_rss *) |
436 | { |
437 | return refcount_read(r: &rss->refcnt); |
438 | } |
439 | |
440 | u32 (struct mlx5e_rss *, enum mlx5_traffic_types tt, |
441 | bool inner) |
442 | { |
443 | struct mlx5e_tir *tir; |
444 | |
445 | WARN_ON(inner && !rss->inner_ft_support); |
446 | tir = rss_get_tir(rss, tt, inner); |
447 | WARN_ON(!tir); |
448 | |
449 | return mlx5e_tir_get_tirn(tir); |
450 | } |
451 | |
452 | /* Fill the "tirn" output parameter. |
453 | * Create the requested TIR if it's its first usage. |
454 | */ |
455 | int (struct mlx5e_rss *, |
456 | enum mlx5_traffic_types tt, |
457 | const struct mlx5e_packet_merge_param *init_pkt_merge_param, |
458 | bool inner, u32 *tirn) |
459 | { |
460 | struct mlx5e_tir *tir; |
461 | |
462 | tir = rss_get_tir(rss, tt, inner); |
463 | if (!tir) { /* TIR doesn't exist, create one */ |
464 | int err; |
465 | |
466 | err = mlx5e_rss_create_tir(rss, tt, init_pkt_merge_param, inner); |
467 | if (err) |
468 | return err; |
469 | tir = rss_get_tir(rss, tt, inner); |
470 | } |
471 | |
472 | *tirn = mlx5e_tir_get_tirn(tir); |
473 | return 0; |
474 | } |
475 | |
476 | static int (struct mlx5e_rss *, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) |
477 | { |
478 | int err; |
479 | |
480 | err = mlx5e_rqt_redirect_indir(rqt: &rss->rqt, rqns, vhca_ids, num_rqns, hfunc: rss->hash.hfunc, |
481 | indir: &rss->indir); |
482 | if (err) |
483 | mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n" , |
484 | mlx5e_rqt_get_rqtn(&rss->rqt), err); |
485 | return err; |
486 | } |
487 | |
488 | void (struct mlx5e_rss *, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) |
489 | { |
490 | rss->enabled = true; |
491 | mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns); |
492 | } |
493 | |
494 | void (struct mlx5e_rss *) |
495 | { |
496 | int err; |
497 | |
498 | rss->enabled = false; |
499 | err = mlx5e_rqt_redirect_direct(rqt: &rss->rqt, rqn: rss->drop_rqn, NULL); |
500 | if (err) |
501 | mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n" , |
502 | mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err); |
503 | } |
504 | |
505 | int (struct mlx5e_rss *, |
506 | struct mlx5e_packet_merge_param *pkt_merge_param) |
507 | { |
508 | struct mlx5e_tir_builder *builder; |
509 | enum mlx5_traffic_types tt; |
510 | int err, final_err; |
511 | |
512 | builder = mlx5e_tir_builder_alloc(modify: true); |
513 | if (!builder) |
514 | return -ENOMEM; |
515 | |
516 | mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param); |
517 | |
518 | final_err = 0; |
519 | |
520 | for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { |
521 | struct mlx5e_tir *tir; |
522 | |
523 | tir = rss_get_tir(rss, tt, inner: false); |
524 | if (!tir) |
525 | goto inner_tir; |
526 | err = mlx5e_tir_modify(tir, builder); |
527 | if (err) { |
528 | mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of indirect TIR %#x for traffic type %d: err = %d\n" , |
529 | mlx5e_tir_get_tirn(tir), tt, err); |
530 | if (!final_err) |
531 | final_err = err; |
532 | } |
533 | |
534 | inner_tir: |
535 | if (!rss->inner_ft_support) |
536 | continue; |
537 | |
538 | tir = rss_get_tir(rss, tt, inner: true); |
539 | if (!tir) |
540 | continue; |
541 | err = mlx5e_tir_modify(tir, builder); |
542 | if (err) { |
543 | mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of inner indirect TIR %#x for traffic type %d: err = %d\n" , |
544 | mlx5e_tir_get_tirn(tir), tt, err); |
545 | if (!final_err) |
546 | final_err = err; |
547 | } |
548 | } |
549 | |
550 | mlx5e_tir_builder_free(builder); |
551 | return final_err; |
552 | } |
553 | |
554 | int (struct mlx5e_rss *, u32 *indir, u8 *key, u8 *hfunc) |
555 | { |
556 | if (indir) |
557 | memcpy(indir, rss->indir.table, |
558 | rss->indir.actual_table_size * sizeof(*rss->indir.table)); |
559 | |
560 | if (key) |
561 | memcpy(key, rss->hash.toeplitz_hash_key, |
562 | sizeof(rss->hash.toeplitz_hash_key)); |
563 | |
564 | if (hfunc) |
565 | *hfunc = rss->hash.hfunc; |
566 | |
567 | return 0; |
568 | } |
569 | |
570 | int (struct mlx5e_rss *, const u32 *indir, |
571 | const u8 *key, const u8 *hfunc, |
572 | u32 *rqns, u32 *vhca_ids, unsigned int num_rqns) |
573 | { |
574 | bool changed_indir = false; |
575 | bool changed_hash = false; |
576 | struct mlx5e_rss *; |
577 | int err = 0; |
578 | |
579 | old_rss = mlx5e_rss_init_copy(from: rss); |
580 | if (IS_ERR(ptr: old_rss)) |
581 | return PTR_ERR(ptr: old_rss); |
582 | |
583 | if (hfunc && *hfunc != rss->hash.hfunc) { |
584 | switch (*hfunc) { |
585 | case ETH_RSS_HASH_XOR: |
586 | case ETH_RSS_HASH_TOP: |
587 | break; |
588 | default: |
589 | err = -EINVAL; |
590 | goto out; |
591 | } |
592 | changed_hash = true; |
593 | changed_indir = true; |
594 | rss->hash.hfunc = *hfunc; |
595 | } |
596 | |
597 | if (key) { |
598 | if (rss->hash.hfunc == ETH_RSS_HASH_TOP) |
599 | changed_hash = true; |
600 | memcpy(rss->hash.toeplitz_hash_key, key, |
601 | sizeof(rss->hash.toeplitz_hash_key)); |
602 | } |
603 | |
604 | if (indir) { |
605 | changed_indir = true; |
606 | |
607 | memcpy(rss->indir.table, indir, |
608 | rss->indir.actual_table_size * sizeof(*rss->indir.table)); |
609 | } |
610 | |
611 | if (changed_indir && rss->enabled) { |
612 | err = mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns); |
613 | if (err) { |
614 | mlx5e_rss_copy(to: rss, from: old_rss); |
615 | goto out; |
616 | } |
617 | } |
618 | |
619 | if (changed_hash) |
620 | mlx5e_rss_update_tirs(rss); |
621 | |
622 | out: |
623 | mlx5e_rss_params_indir_cleanup(indir: &old_rss->indir); |
624 | kvfree(addr: old_rss); |
625 | |
626 | return err; |
627 | } |
628 | |
629 | struct mlx5e_rss_params_hash (struct mlx5e_rss *) |
630 | { |
631 | return rss->hash; |
632 | } |
633 | |
634 | u8 (struct mlx5e_rss *, enum mlx5_traffic_types tt) |
635 | { |
636 | return rss->rx_hash_fields[tt]; |
637 | } |
638 | |
639 | int (struct mlx5e_rss *, enum mlx5_traffic_types tt, |
640 | u8 rx_hash_fields) |
641 | { |
642 | u8 old_rx_hash_fields; |
643 | int err; |
644 | |
645 | old_rx_hash_fields = rss->rx_hash_fields[tt]; |
646 | |
647 | if (old_rx_hash_fields == rx_hash_fields) |
648 | return 0; |
649 | |
650 | rss->rx_hash_fields[tt] = rx_hash_fields; |
651 | |
652 | err = mlx5e_rss_update_tir(rss, tt, inner: false); |
653 | if (err) { |
654 | rss->rx_hash_fields[tt] = old_rx_hash_fields; |
655 | mlx5e_rss_warn(rss->mdev, |
656 | "Failed to update RSS hash fields of indirect TIR for traffic type %d: err = %d\n" , |
657 | tt, err); |
658 | return err; |
659 | } |
660 | |
661 | if (!(rss->inner_ft_support)) |
662 | return 0; |
663 | |
664 | err = mlx5e_rss_update_tir(rss, tt, inner: true); |
665 | if (err) { |
666 | /* Partial update happened. Try to revert - it may fail too, but |
667 | * there is nothing more we can do. |
668 | */ |
669 | rss->rx_hash_fields[tt] = old_rx_hash_fields; |
670 | mlx5e_rss_warn(rss->mdev, |
671 | "Failed to update RSS hash fields of inner indirect TIR for traffic type %d: err = %d\n" , |
672 | tt, err); |
673 | if (mlx5e_rss_update_tir(rss, tt, inner: false)) |
674 | mlx5e_rss_warn(rss->mdev, |
675 | "Partial update of RSS hash fields happened: failed to revert indirect TIR for traffic type %d to the old values\n" , |
676 | tt); |
677 | } |
678 | |
679 | return err; |
680 | } |
681 | |
682 | void (struct mlx5e_rss *, unsigned int nch) |
683 | { |
684 | mlx5e_rss_params_indir_init_uniform(indir: &rss->indir, num_channels: nch); |
685 | } |
686 | |