1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3
4#include "en/fs_tt_redirect.h"
5#include "fs_core.h"
6#include "mlx5_core.h"
7
8enum fs_udp_type {
9 FS_IPV4_UDP,
10 FS_IPV6_UDP,
11 FS_UDP_NUM_TYPES,
12};
13
14struct mlx5e_fs_udp {
15 struct mlx5e_flow_table tables[FS_UDP_NUM_TYPES];
16 struct mlx5_flow_handle *default_rules[FS_UDP_NUM_TYPES];
17 int ref_cnt;
18};
19
20struct mlx5e_fs_any {
21 struct mlx5e_flow_table table;
22 struct mlx5_flow_handle *default_rule;
23 int ref_cnt;
24};
25
26static char *fs_udp_type2str(enum fs_udp_type i)
27{
28 switch (i) {
29 case FS_IPV4_UDP:
30 return "UDP v4";
31 default: /* FS_IPV6_UDP */
32 return "UDP v6";
33 }
34}
35
36static enum mlx5_traffic_types fs_udp2tt(enum fs_udp_type i)
37{
38 switch (i) {
39 case FS_IPV4_UDP:
40 return MLX5_TT_IPV4_UDP;
41 default: /* FS_IPV6_UDP */
42 return MLX5_TT_IPV6_UDP;
43 }
44}
45
46static enum fs_udp_type tt2fs_udp(enum mlx5_traffic_types i)
47{
48 switch (i) {
49 case MLX5_TT_IPV4_UDP:
50 return FS_IPV4_UDP;
51 case MLX5_TT_IPV6_UDP:
52 return FS_IPV6_UDP;
53 default:
54 return FS_UDP_NUM_TYPES;
55 }
56}
57
58void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule)
59{
60 mlx5_del_flow_rules(fr: rule);
61}
62
63static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type type,
64 u16 udp_dport)
65{
66 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
67 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
68 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
69 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
70 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
71 type == FS_IPV4_UDP ? 4 : 6);
72 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
73 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, udp_dport);
74}
75
76struct mlx5_flow_handle *
77mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_flow_steering *fs,
78 enum mlx5_traffic_types ttc_type,
79 u32 tir_num, u16 d_port)
80{
81 struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
82 enum fs_udp_type type = tt2fs_udp(i: ttc_type);
83 struct mlx5_flow_destination dest = {};
84 struct mlx5_flow_table *ft = NULL;
85 MLX5_DECLARE_FLOW_ACT(flow_act);
86 struct mlx5_flow_handle *rule;
87 struct mlx5_flow_spec *spec;
88 int err;
89
90 if (type == FS_UDP_NUM_TYPES)
91 return ERR_PTR(error: -EINVAL);
92
93 spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
94 if (!spec)
95 return ERR_PTR(error: -ENOMEM);
96
97 ft = fs_udp->tables[type].t;
98
99 fs_udp_set_dport_flow(spec, type, udp_dport: d_port);
100 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
101 dest.tir_num = tir_num;
102
103 rule = mlx5_add_flow_rules(ft, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
104 kvfree(addr: spec);
105
106 if (IS_ERR(ptr: rule)) {
107 err = PTR_ERR(ptr: rule);
108 fs_err(fs, "%s: add %s rule failed, err %d\n",
109 __func__, fs_udp_type2str(type), err);
110 }
111 return rule;
112}
113
114static int fs_udp_add_default_rule(struct mlx5e_flow_steering *fs, enum fs_udp_type type)
115{
116 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false);
117 struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
118 struct mlx5e_flow_table *fs_udp_t;
119 struct mlx5_flow_destination dest;
120 MLX5_DECLARE_FLOW_ACT(flow_act);
121 struct mlx5_flow_handle *rule;
122 int err;
123
124 fs_udp_t = &fs_udp->tables[type];
125
126 dest = mlx5_ttc_get_default_dest(ttc, type: fs_udp2tt(i: type));
127 rule = mlx5_add_flow_rules(ft: fs_udp_t->t, NULL, flow_act: &flow_act, dest: &dest, num_dest: 1);
128 if (IS_ERR(ptr: rule)) {
129 err = PTR_ERR(ptr: rule);
130 fs_err(fs, "%s: add default rule failed, fs type=%d, err %d\n",
131 __func__, type, err);
132 return err;
133 }
134
135 fs_udp->default_rules[type] = rule;
136 return 0;
137}
138
139#define MLX5E_FS_UDP_NUM_GROUPS (2)
140#define MLX5E_FS_UDP_GROUP1_SIZE (BIT(16))
141#define MLX5E_FS_UDP_GROUP2_SIZE (BIT(0))
142#define MLX5E_FS_UDP_TABLE_SIZE (MLX5E_FS_UDP_GROUP1_SIZE +\
143 MLX5E_FS_UDP_GROUP2_SIZE)
144static int fs_udp_create_groups(struct mlx5e_flow_table *ft, enum fs_udp_type type)
145{
146 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
147 void *outer_headers_c;
148 int ix = 0;
149 u32 *in;
150 int err;
151 u8 *mc;
152
153 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, size: sizeof(*ft->g), GFP_KERNEL);
154 in = kvzalloc(size: inlen, GFP_KERNEL);
155 if (!in || !ft->g) {
156 kfree(objp: ft->g);
157 ft->g = NULL;
158 kvfree(addr: in);
159 return -ENOMEM;
160 }
161
162 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
163 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
164 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
165 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
166
167 switch (type) {
168 case FS_IPV4_UDP:
169 case FS_IPV6_UDP:
170 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
171 break;
172 default:
173 err = -EINVAL;
174 goto out;
175 }
176 /* Match on udp protocol, Ipv4/6 and dport */
177 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
178 MLX5_SET_CFG(in, start_flow_index, ix);
179 ix += MLX5E_FS_UDP_GROUP1_SIZE;
180 MLX5_SET_CFG(in, end_flow_index, ix - 1);
181 ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
182 if (IS_ERR(ptr: ft->g[ft->num_groups]))
183 goto err;
184 ft->num_groups++;
185
186 /* Default Flow Group */
187 memset(in, 0, inlen);
188 MLX5_SET_CFG(in, start_flow_index, ix);
189 ix += MLX5E_FS_UDP_GROUP2_SIZE;
190 MLX5_SET_CFG(in, end_flow_index, ix - 1);
191 ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
192 if (IS_ERR(ptr: ft->g[ft->num_groups]))
193 goto err;
194 ft->num_groups++;
195
196 kvfree(addr: in);
197 return 0;
198
199err:
200 err = PTR_ERR(ptr: ft->g[ft->num_groups]);
201 ft->g[ft->num_groups] = NULL;
202out:
203 kvfree(addr: in);
204
205 return err;
206}
207
208static int fs_udp_create_table(struct mlx5e_flow_steering *fs, enum fs_udp_type type)
209{
210 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, egress: false);
211 struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
212 struct mlx5_flow_table_attr ft_attr = {};
213 struct mlx5e_flow_table *ft;
214 int err;
215
216 ft = &fs_udp->tables[type];
217 ft->num_groups = 0;
218
219 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
220 ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL;
221 ft_attr.prio = MLX5E_NIC_PRIO;
222
223 ft->t = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
224 if (IS_ERR(ptr: ft->t)) {
225 err = PTR_ERR(ptr: ft->t);
226 ft->t = NULL;
227 return err;
228 }
229
230 mlx5_core_dbg(mlx5e_fs_get_mdev(fs), "Created fs %s table id %u level %u\n",
231 fs_udp_type2str(type), ft->t->id, ft->t->level);
232
233 err = fs_udp_create_groups(ft, type);
234 if (err)
235 goto err;
236
237 err = fs_udp_add_default_rule(fs, type);
238 if (err)
239 goto err;
240
241 return 0;
242
243err:
244 mlx5e_destroy_flow_table(ft);
245 return err;
246}
247
248static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i)
249{
250 if (IS_ERR_OR_NULL(ptr: fs_udp->tables[i].t))
251 return;
252
253 mlx5_del_flow_rules(fr: fs_udp->default_rules[i]);
254 mlx5e_destroy_flow_table(ft: &fs_udp->tables[i]);
255 fs_udp->tables[i].t = NULL;
256}
257
258static int fs_udp_disable(struct mlx5e_flow_steering *fs)
259{
260 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false);
261 int err, i;
262
263 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
264 /* Modify ttc rules destination to point back to the indir TIRs */
265 err = mlx5_ttc_fwd_default_dest(ttc, type: fs_udp2tt(i));
266 if (err) {
267 fs_err(fs, "%s: modify ttc[%d] default destination failed, err(%d)\n",
268 __func__, fs_udp2tt(i), err);
269 return err;
270 }
271 }
272
273 return 0;
274}
275
276static int fs_udp_enable(struct mlx5e_flow_steering *fs)
277{
278 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false);
279 struct mlx5e_fs_udp *udp = mlx5e_fs_get_udp(fs);
280 struct mlx5_flow_destination dest = {};
281 int err, i;
282
283 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
284 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
285 dest.ft = udp->tables[i].t;
286
287 /* Modify ttc rules destination to point on the accel_fs FTs */
288 err = mlx5_ttc_fwd_dest(ttc, type: fs_udp2tt(i), new_dest: &dest);
289 if (err) {
290 fs_err(fs, "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
291 __func__, fs_udp2tt(i), err);
292 return err;
293 }
294 }
295 return 0;
296}
297
298void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_flow_steering *fs)
299{
300 struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
301 int i;
302
303 if (!fs_udp)
304 return;
305
306 if (--fs_udp->ref_cnt)
307 return;
308
309 fs_udp_disable(fs);
310
311 for (i = 0; i < FS_UDP_NUM_TYPES; i++)
312 fs_udp_destroy_table(fs_udp, i);
313
314 kfree(objp: fs_udp);
315 mlx5e_fs_set_udp(fs, NULL);
316}
317
318int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_flow_steering *fs)
319{
320 struct mlx5e_fs_udp *udp = mlx5e_fs_get_udp(fs);
321 int i, err;
322
323 if (udp) {
324 udp->ref_cnt++;
325 return 0;
326 }
327
328 udp = kzalloc(size: sizeof(*udp), GFP_KERNEL);
329 if (!udp)
330 return -ENOMEM;
331 mlx5e_fs_set_udp(fs, udp);
332
333 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
334 err = fs_udp_create_table(fs, type: i);
335 if (err)
336 goto err_destroy_tables;
337 }
338
339 err = fs_udp_enable(fs);
340 if (err)
341 goto err_destroy_tables;
342
343 udp->ref_cnt = 1;
344
345 return 0;
346
347err_destroy_tables:
348 while (--i >= 0)
349 fs_udp_destroy_table(fs_udp: udp, i);
350
351 kfree(objp: udp);
352 mlx5e_fs_set_udp(fs, NULL);
353 return err;
354}
355
356static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_type)
357{
358 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
359 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
360 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ether_type);
361}
362
363struct mlx5_flow_handle *
364mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_flow_steering *fs,
365 u32 tir_num, u16 ether_type)
366{
367 struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
368 struct mlx5_flow_destination dest = {};
369 struct mlx5_flow_table *ft = NULL;
370 MLX5_DECLARE_FLOW_ACT(flow_act);
371 struct mlx5_flow_handle *rule;
372 struct mlx5_flow_spec *spec;
373 int err;
374
375 spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL);
376 if (!spec)
377 return ERR_PTR(error: -ENOMEM);
378
379 ft = fs_any->table.t;
380
381 fs_any_set_ethertype_flow(spec, ether_type);
382 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
383 dest.tir_num = tir_num;
384
385 rule = mlx5_add_flow_rules(ft, spec, flow_act: &flow_act, dest: &dest, num_dest: 1);
386 kvfree(addr: spec);
387
388 if (IS_ERR(ptr: rule)) {
389 err = PTR_ERR(ptr: rule);
390 fs_err(fs, "%s: add ANY rule failed, err %d\n",
391 __func__, err);
392 }
393 return rule;
394}
395
396static int fs_any_add_default_rule(struct mlx5e_flow_steering *fs)
397{
398 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false);
399 struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
400 struct mlx5e_flow_table *fs_any_t;
401 struct mlx5_flow_destination dest;
402 MLX5_DECLARE_FLOW_ACT(flow_act);
403 struct mlx5_flow_handle *rule;
404 int err;
405
406 fs_any_t = &fs_any->table;
407 dest = mlx5_ttc_get_default_dest(ttc, type: MLX5_TT_ANY);
408 rule = mlx5_add_flow_rules(ft: fs_any_t->t, NULL, flow_act: &flow_act, dest: &dest, num_dest: 1);
409 if (IS_ERR(ptr: rule)) {
410 err = PTR_ERR(ptr: rule);
411 fs_err(fs, "%s: add default rule failed, fs type=ANY, err %d\n",
412 __func__, err);
413 return err;
414 }
415
416 fs_any->default_rule = rule;
417 return 0;
418}
419
420#define MLX5E_FS_ANY_NUM_GROUPS (2)
421#define MLX5E_FS_ANY_GROUP1_SIZE (BIT(16))
422#define MLX5E_FS_ANY_GROUP2_SIZE (BIT(0))
423#define MLX5E_FS_ANY_TABLE_SIZE (MLX5E_FS_ANY_GROUP1_SIZE +\
424 MLX5E_FS_ANY_GROUP2_SIZE)
425
426static int fs_any_create_groups(struct mlx5e_flow_table *ft)
427{
428 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
429 void *outer_headers_c;
430 int ix = 0;
431 u32 *in;
432 int err;
433 u8 *mc;
434
435 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, size: sizeof(*ft->g), GFP_KERNEL);
436 in = kvzalloc(size: inlen, GFP_KERNEL);
437 if (!in || !ft->g) {
438 kfree(objp: ft->g);
439 ft->g = NULL;
440 kvfree(addr: in);
441 return -ENOMEM;
442 }
443
444 /* Match on ethertype */
445 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
446 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
447 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
448 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
449 MLX5_SET_CFG(in, start_flow_index, ix);
450 ix += MLX5E_FS_ANY_GROUP1_SIZE;
451 MLX5_SET_CFG(in, end_flow_index, ix - 1);
452 ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
453 if (IS_ERR(ptr: ft->g[ft->num_groups]))
454 goto err;
455 ft->num_groups++;
456
457 /* Default Flow Group */
458 memset(in, 0, inlen);
459 MLX5_SET_CFG(in, start_flow_index, ix);
460 ix += MLX5E_FS_ANY_GROUP2_SIZE;
461 MLX5_SET_CFG(in, end_flow_index, ix - 1);
462 ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in);
463 if (IS_ERR(ptr: ft->g[ft->num_groups]))
464 goto err;
465 ft->num_groups++;
466
467 kvfree(addr: in);
468 return 0;
469
470err:
471 err = PTR_ERR(ptr: ft->g[ft->num_groups]);
472 ft->g[ft->num_groups] = NULL;
473 kvfree(addr: in);
474
475 return err;
476}
477
478static int fs_any_create_table(struct mlx5e_flow_steering *fs)
479{
480 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, egress: false);
481 struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
482 struct mlx5e_flow_table *ft = &fs_any->table;
483 struct mlx5_flow_table_attr ft_attr = {};
484 int err;
485
486 ft->num_groups = 0;
487
488 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
489 ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL;
490 ft_attr.prio = MLX5E_NIC_PRIO;
491
492 ft->t = mlx5_create_flow_table(ns, ft_attr: &ft_attr);
493 if (IS_ERR(ptr: ft->t)) {
494 err = PTR_ERR(ptr: ft->t);
495 ft->t = NULL;
496 return err;
497 }
498
499 mlx5_core_dbg(mlx5e_fs_get_mdev(fs), "Created fs ANY table id %u level %u\n",
500 ft->t->id, ft->t->level);
501
502 err = fs_any_create_groups(ft);
503 if (err)
504 goto err;
505
506 err = fs_any_add_default_rule(fs);
507 if (err)
508 goto err;
509
510 return 0;
511
512err:
513 mlx5e_destroy_flow_table(ft);
514 return err;
515}
516
517static int fs_any_disable(struct mlx5e_flow_steering *fs)
518{
519 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false);
520 int err;
521
522 /* Modify ttc rules destination to point back to the indir TIRs */
523 err = mlx5_ttc_fwd_default_dest(ttc, type: MLX5_TT_ANY);
524 if (err) {
525 fs_err(fs,
526 "%s: modify ttc[%d] default destination failed, err(%d)\n",
527 __func__, MLX5_TT_ANY, err);
528 return err;
529 }
530 return 0;
531}
532
533static int fs_any_enable(struct mlx5e_flow_steering *fs)
534{
535 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false);
536 struct mlx5e_fs_any *any = mlx5e_fs_get_any(fs);
537 struct mlx5_flow_destination dest = {};
538 int err;
539
540 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
541 dest.ft = any->table.t;
542
543 /* Modify ttc rules destination to point on the accel_fs FTs */
544 err = mlx5_ttc_fwd_dest(ttc, type: MLX5_TT_ANY, new_dest: &dest);
545 if (err) {
546 fs_err(fs,
547 "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
548 __func__, MLX5_TT_ANY, err);
549 return err;
550 }
551 return 0;
552}
553
554static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any)
555{
556 if (IS_ERR_OR_NULL(ptr: fs_any->table.t))
557 return;
558
559 mlx5_del_flow_rules(fr: fs_any->default_rule);
560 mlx5e_destroy_flow_table(ft: &fs_any->table);
561 fs_any->table.t = NULL;
562}
563
564void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_flow_steering *fs)
565{
566 struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
567
568 if (!fs_any)
569 return;
570
571 if (--fs_any->ref_cnt)
572 return;
573
574 fs_any_disable(fs);
575
576 fs_any_destroy_table(fs_any);
577
578 kfree(objp: fs_any);
579 mlx5e_fs_set_any(fs, NULL);
580}
581
582int mlx5e_fs_tt_redirect_any_create(struct mlx5e_flow_steering *fs)
583{
584 struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
585 int err;
586
587 if (fs_any) {
588 fs_any->ref_cnt++;
589 return 0;
590 }
591
592 fs_any = kzalloc(size: sizeof(*fs_any), GFP_KERNEL);
593 if (!fs_any)
594 return -ENOMEM;
595 mlx5e_fs_set_any(fs, any: fs_any);
596
597 err = fs_any_create_table(fs);
598 if (err)
599 goto err_free_any;
600
601 err = fs_any_enable(fs);
602 if (err)
603 goto err_destroy_table;
604
605 fs_any->ref_cnt = 1;
606
607 return 0;
608
609err_destroy_table:
610 fs_any_destroy_table(fs_any);
611err_free_any:
612 mlx5e_fs_set_any(fs, NULL);
613 kfree(objp: fs_any);
614 return err;
615}
616

source code of linux/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c