1 | /* |
2 | * Copyright (c) 2016, Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #include <linux/hash.h> |
34 | #include <linux/mlx5/fs.h> |
35 | #include <linux/ip.h> |
36 | #include <linux/ipv6.h> |
37 | #include <net/rps.h> |
38 | #include "en.h" |
39 | |
40 | #define ARFS_HASH_SHIFT BITS_PER_BYTE |
41 | #define ARFS_HASH_SIZE BIT(BITS_PER_BYTE) |
42 | |
43 | struct arfs_table { |
44 | struct mlx5e_flow_table ft; |
45 | struct mlx5_flow_handle *default_rule; |
46 | struct hlist_head rules_hash[ARFS_HASH_SIZE]; |
47 | }; |
48 | |
49 | enum { |
50 | MLX5E_ARFS_STATE_ENABLED, |
51 | }; |
52 | |
53 | enum arfs_type { |
54 | ARFS_IPV4_TCP, |
55 | ARFS_IPV6_TCP, |
56 | ARFS_IPV4_UDP, |
57 | ARFS_IPV6_UDP, |
58 | ARFS_NUM_TYPES, |
59 | }; |
60 | |
61 | struct mlx5e_arfs_tables { |
62 | struct arfs_table arfs_tables[ARFS_NUM_TYPES]; |
63 | /* Protect aRFS rules list */ |
64 | spinlock_t arfs_lock; |
65 | int last_filter_id; |
66 | struct workqueue_struct *wq; |
67 | unsigned long state; |
68 | }; |
69 | |
70 | struct arfs_tuple { |
71 | __be16 etype; |
72 | u8 ip_proto; |
73 | union { |
74 | __be32 src_ipv4; |
75 | struct in6_addr src_ipv6; |
76 | }; |
77 | union { |
78 | __be32 dst_ipv4; |
79 | struct in6_addr dst_ipv6; |
80 | }; |
81 | __be16 src_port; |
82 | __be16 dst_port; |
83 | }; |
84 | |
85 | struct arfs_rule { |
86 | struct mlx5e_priv *priv; |
87 | struct work_struct arfs_work; |
88 | struct mlx5_flow_handle *rule; |
89 | struct hlist_node hlist; |
90 | int rxq; |
91 | /* Flow ID passed to ndo_rx_flow_steer */ |
92 | int flow_id; |
93 | /* Filter ID returned by ndo_rx_flow_steer */ |
94 | int filter_id; |
95 | struct arfs_tuple tuple; |
96 | }; |
97 | |
98 | #define mlx5e_for_each_arfs_rule(hn, tmp, arfs_tables, i, j) \ |
99 | for (i = 0; i < ARFS_NUM_TYPES; i++) \ |
100 | mlx5e_for_each_hash_arfs_rule(hn, tmp, arfs_tables[i].rules_hash, j) |
101 | |
102 | #define mlx5e_for_each_hash_arfs_rule(hn, tmp, hash, j) \ |
103 | for (j = 0; j < ARFS_HASH_SIZE; j++) \ |
104 | hlist_for_each_entry_safe(hn, tmp, &hash[j], hlist) |
105 | |
106 | static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type) |
107 | { |
108 | switch (type) { |
109 | case ARFS_IPV4_TCP: |
110 | return MLX5_TT_IPV4_TCP; |
111 | case ARFS_IPV4_UDP: |
112 | return MLX5_TT_IPV4_UDP; |
113 | case ARFS_IPV6_TCP: |
114 | return MLX5_TT_IPV6_TCP; |
115 | case ARFS_IPV6_UDP: |
116 | return MLX5_TT_IPV6_UDP; |
117 | default: |
118 | return -EINVAL; |
119 | } |
120 | } |
121 | |
122 | static int arfs_disable(struct mlx5e_flow_steering *fs) |
123 | { |
124 | struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false); |
125 | int err, i; |
126 | |
127 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
128 | /* Modify ttc rules destination back to their default */ |
129 | err = mlx5_ttc_fwd_default_dest(ttc, type: arfs_get_tt(type: i)); |
130 | if (err) { |
131 | fs_err(fs, |
132 | "%s: modify ttc[%d] default destination failed, err(%d)\n" , |
133 | __func__, arfs_get_tt(i), err); |
134 | return err; |
135 | } |
136 | } |
137 | return 0; |
138 | } |
139 | |
140 | static void arfs_del_rules(struct mlx5e_flow_steering *fs); |
141 | |
142 | int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs) |
143 | { |
144 | /* Moving to switchdev mode, fs->arfs is freed by mlx5e_nic_profile |
145 | * cleanup_rx callback and it is not recreated when |
146 | * mlx5e_uplink_rep_profile is loaded as mlx5e_create_flow_steering() |
147 | * is not called by the uplink_rep profile init_rx callback. Thus, if |
148 | * ntuple is set, moving to switchdev flow will enter this function |
149 | * with fs->arfs nullified. |
150 | */ |
151 | if (!mlx5e_fs_get_arfs(fs)) |
152 | return 0; |
153 | |
154 | arfs_del_rules(fs); |
155 | |
156 | return arfs_disable(fs); |
157 | } |
158 | |
159 | int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs) |
160 | { |
161 | struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, inner: false); |
162 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); |
163 | struct mlx5_flow_destination dest = {}; |
164 | int err, i; |
165 | |
166 | dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; |
167 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
168 | dest.ft = arfs->arfs_tables[i].ft.t; |
169 | /* Modify ttc rules destination to point on the aRFS FTs */ |
170 | err = mlx5_ttc_fwd_dest(ttc, type: arfs_get_tt(type: i), new_dest: &dest); |
171 | if (err) { |
172 | fs_err(fs, "%s: modify ttc[%d] dest to arfs, failed err(%d)\n" , |
173 | __func__, arfs_get_tt(i), err); |
174 | arfs_disable(fs); |
175 | return err; |
176 | } |
177 | } |
178 | set_bit(nr: MLX5E_ARFS_STATE_ENABLED, addr: &arfs->state); |
179 | |
180 | return 0; |
181 | } |
182 | |
183 | static void arfs_destroy_table(struct arfs_table *arfs_t) |
184 | { |
185 | mlx5_del_flow_rules(fr: arfs_t->default_rule); |
186 | mlx5e_destroy_flow_table(ft: &arfs_t->ft); |
187 | } |
188 | |
189 | static void _mlx5e_cleanup_tables(struct mlx5e_flow_steering *fs) |
190 | { |
191 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); |
192 | int i; |
193 | |
194 | arfs_del_rules(fs); |
195 | destroy_workqueue(wq: arfs->wq); |
196 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
197 | if (!IS_ERR_OR_NULL(ptr: arfs->arfs_tables[i].ft.t)) |
198 | arfs_destroy_table(arfs_t: &arfs->arfs_tables[i]); |
199 | } |
200 | } |
201 | |
202 | void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple) |
203 | { |
204 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); |
205 | |
206 | if (!ntuple) |
207 | return; |
208 | |
209 | _mlx5e_cleanup_tables(fs); |
210 | mlx5e_fs_set_arfs(fs, NULL); |
211 | kvfree(addr: arfs); |
212 | } |
213 | |
214 | static int arfs_add_default_rule(struct mlx5e_flow_steering *fs, |
215 | struct mlx5e_rx_res *rx_res, |
216 | enum arfs_type type) |
217 | { |
218 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); |
219 | struct arfs_table *arfs_t = &arfs->arfs_tables[type]; |
220 | struct mlx5_flow_destination dest = {}; |
221 | MLX5_DECLARE_FLOW_ACT(flow_act); |
222 | enum mlx5_traffic_types tt; |
223 | int err = 0; |
224 | |
225 | dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
226 | tt = arfs_get_tt(type); |
227 | if (tt == -EINVAL) { |
228 | fs_err(fs, "%s: bad arfs_type: %d\n" , __func__, type); |
229 | return -EINVAL; |
230 | } |
231 | |
232 | /* FIXME: Must use mlx5_ttc_get_default_dest(), |
233 | * but can't since TTC default is not setup yet ! |
234 | */ |
235 | dest.tir_num = mlx5e_rx_res_get_tirn_rss(res: rx_res, tt); |
236 | arfs_t->default_rule = mlx5_add_flow_rules(ft: arfs_t->ft.t, NULL, |
237 | flow_act: &flow_act, |
238 | dest: &dest, num_dest: 1); |
239 | if (IS_ERR(ptr: arfs_t->default_rule)) { |
240 | err = PTR_ERR(ptr: arfs_t->default_rule); |
241 | arfs_t->default_rule = NULL; |
242 | fs_err(fs, "%s: add rule failed, arfs type=%d\n" , __func__, type); |
243 | } |
244 | |
245 | return err; |
246 | } |
247 | |
248 | #define MLX5E_ARFS_NUM_GROUPS 2 |
249 | #define MLX5E_ARFS_GROUP1_SIZE (BIT(16) - 1) |
250 | #define MLX5E_ARFS_GROUP2_SIZE BIT(0) |
251 | #define MLX5E_ARFS_TABLE_SIZE (MLX5E_ARFS_GROUP1_SIZE +\ |
252 | MLX5E_ARFS_GROUP2_SIZE) |
253 | static int arfs_create_groups(struct mlx5e_flow_table *ft, |
254 | enum arfs_type type) |
255 | { |
256 | int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); |
257 | void *; |
258 | int ix = 0; |
259 | u32 *in; |
260 | int err; |
261 | u8 *mc; |
262 | |
263 | ft->g = kcalloc(MLX5E_ARFS_NUM_GROUPS, |
264 | size: sizeof(*ft->g), GFP_KERNEL); |
265 | if (!ft->g) |
266 | return -ENOMEM; |
267 | |
268 | in = kvzalloc(size: inlen, GFP_KERNEL); |
269 | if (!in) { |
270 | err = -ENOMEM; |
271 | goto err_free_g; |
272 | } |
273 | |
274 | mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); |
275 | outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, |
276 | outer_headers); |
277 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype); |
278 | switch (type) { |
279 | case ARFS_IPV4_TCP: |
280 | case ARFS_IPV6_TCP: |
281 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport); |
282 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport); |
283 | break; |
284 | case ARFS_IPV4_UDP: |
285 | case ARFS_IPV6_UDP: |
286 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); |
287 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_sport); |
288 | break; |
289 | default: |
290 | err = -EINVAL; |
291 | goto err_free_in; |
292 | } |
293 | |
294 | switch (type) { |
295 | case ARFS_IPV4_TCP: |
296 | case ARFS_IPV4_UDP: |
297 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, |
298 | src_ipv4_src_ipv6.ipv4_layout.ipv4); |
299 | MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, |
300 | dst_ipv4_dst_ipv6.ipv4_layout.ipv4); |
301 | break; |
302 | case ARFS_IPV6_TCP: |
303 | case ARFS_IPV6_UDP: |
304 | memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, |
305 | src_ipv4_src_ipv6.ipv6_layout.ipv6), |
306 | 0xff, 16); |
307 | memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c, |
308 | dst_ipv4_dst_ipv6.ipv6_layout.ipv6), |
309 | 0xff, 16); |
310 | break; |
311 | default: |
312 | err = -EINVAL; |
313 | goto err_free_in; |
314 | } |
315 | |
316 | MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); |
317 | MLX5_SET_CFG(in, start_flow_index, ix); |
318 | ix += MLX5E_ARFS_GROUP1_SIZE; |
319 | MLX5_SET_CFG(in, end_flow_index, ix - 1); |
320 | ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in); |
321 | if (IS_ERR(ptr: ft->g[ft->num_groups])) |
322 | goto err_clean_group; |
323 | ft->num_groups++; |
324 | |
325 | memset(in, 0, inlen); |
326 | MLX5_SET_CFG(in, start_flow_index, ix); |
327 | ix += MLX5E_ARFS_GROUP2_SIZE; |
328 | MLX5_SET_CFG(in, end_flow_index, ix - 1); |
329 | ft->g[ft->num_groups] = mlx5_create_flow_group(ft: ft->t, in); |
330 | if (IS_ERR(ptr: ft->g[ft->num_groups])) |
331 | goto err_clean_group; |
332 | ft->num_groups++; |
333 | |
334 | kvfree(addr: in); |
335 | return 0; |
336 | |
337 | err_clean_group: |
338 | err = PTR_ERR(ptr: ft->g[ft->num_groups]); |
339 | ft->g[ft->num_groups] = NULL; |
340 | err_free_in: |
341 | kvfree(addr: in); |
342 | err_free_g: |
343 | kfree(objp: ft->g); |
344 | ft->g = NULL; |
345 | return err; |
346 | } |
347 | |
348 | static int arfs_create_table(struct mlx5e_flow_steering *fs, |
349 | struct mlx5e_rx_res *rx_res, |
350 | enum arfs_type type) |
351 | { |
352 | struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, egress: false); |
353 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); |
354 | struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; |
355 | struct mlx5_flow_table_attr ft_attr = {}; |
356 | int err; |
357 | |
358 | ft->num_groups = 0; |
359 | |
360 | ft_attr.max_fte = MLX5E_ARFS_TABLE_SIZE; |
361 | ft_attr.level = MLX5E_ARFS_FT_LEVEL; |
362 | ft_attr.prio = MLX5E_NIC_PRIO; |
363 | |
364 | ft->t = mlx5_create_flow_table(ns, ft_attr: &ft_attr); |
365 | if (IS_ERR(ptr: ft->t)) { |
366 | err = PTR_ERR(ptr: ft->t); |
367 | ft->t = NULL; |
368 | return err; |
369 | } |
370 | |
371 | err = arfs_create_groups(ft, type); |
372 | if (err) |
373 | goto err; |
374 | |
375 | err = arfs_add_default_rule(fs, rx_res, type); |
376 | if (err) |
377 | goto err; |
378 | |
379 | return 0; |
380 | err: |
381 | mlx5e_destroy_flow_table(ft); |
382 | return err; |
383 | } |
384 | |
385 | int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs, |
386 | struct mlx5e_rx_res *rx_res, bool ntuple) |
387 | { |
388 | struct mlx5e_arfs_tables *arfs; |
389 | int err = -ENOMEM; |
390 | int i; |
391 | |
392 | if (!ntuple) |
393 | return 0; |
394 | |
395 | arfs = kvzalloc(size: sizeof(*arfs), GFP_KERNEL); |
396 | if (!arfs) |
397 | return -ENOMEM; |
398 | |
399 | spin_lock_init(&arfs->arfs_lock); |
400 | arfs->wq = create_singlethread_workqueue("mlx5e_arfs" ); |
401 | if (!arfs->wq) |
402 | goto err; |
403 | |
404 | mlx5e_fs_set_arfs(fs, arfs); |
405 | |
406 | for (i = 0; i < ARFS_NUM_TYPES; i++) { |
407 | err = arfs_create_table(fs, rx_res, type: i); |
408 | if (err) |
409 | goto err_des; |
410 | } |
411 | return 0; |
412 | |
413 | err_des: |
414 | _mlx5e_cleanup_tables(fs); |
415 | err: |
416 | mlx5e_fs_set_arfs(fs, NULL); |
417 | kvfree(addr: arfs); |
418 | return err; |
419 | } |
420 | |
421 | #define MLX5E_ARFS_EXPIRY_QUOTA 60 |
422 | |
423 | static void arfs_may_expire_flow(struct mlx5e_priv *priv) |
424 | { |
425 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs: priv->fs); |
426 | struct arfs_rule *arfs_rule; |
427 | struct hlist_node *htmp; |
428 | HLIST_HEAD(del_list); |
429 | int quota = 0; |
430 | int i; |
431 | int j; |
432 | |
433 | spin_lock_bh(lock: &arfs->arfs_lock); |
434 | mlx5e_for_each_arfs_rule(arfs_rule, htmp, arfs->arfs_tables, i, j) { |
435 | if (!work_pending(&arfs_rule->arfs_work) && |
436 | rps_may_expire_flow(dev: priv->netdev, |
437 | rxq_index: arfs_rule->rxq, flow_id: arfs_rule->flow_id, |
438 | filter_id: arfs_rule->filter_id)) { |
439 | hlist_del_init(n: &arfs_rule->hlist); |
440 | hlist_add_head(n: &arfs_rule->hlist, h: &del_list); |
441 | if (quota++ > MLX5E_ARFS_EXPIRY_QUOTA) |
442 | break; |
443 | } |
444 | } |
445 | spin_unlock_bh(lock: &arfs->arfs_lock); |
446 | hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { |
447 | if (arfs_rule->rule) { |
448 | mlx5_del_flow_rules(fr: arfs_rule->rule); |
449 | priv->channel_stats[arfs_rule->rxq]->rq.arfs_expired++; |
450 | } |
451 | hlist_del(n: &arfs_rule->hlist); |
452 | kfree(objp: arfs_rule); |
453 | } |
454 | } |
455 | |
456 | static void arfs_del_rules(struct mlx5e_flow_steering *fs) |
457 | { |
458 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs); |
459 | struct hlist_node *htmp; |
460 | struct arfs_rule *rule; |
461 | HLIST_HEAD(del_list); |
462 | int i; |
463 | int j; |
464 | |
465 | clear_bit(nr: MLX5E_ARFS_STATE_ENABLED, addr: &arfs->state); |
466 | |
467 | spin_lock_bh(lock: &arfs->arfs_lock); |
468 | mlx5e_for_each_arfs_rule(rule, htmp, arfs->arfs_tables, i, j) { |
469 | hlist_del_init(n: &rule->hlist); |
470 | hlist_add_head(n: &rule->hlist, h: &del_list); |
471 | } |
472 | spin_unlock_bh(lock: &arfs->arfs_lock); |
473 | |
474 | hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { |
475 | cancel_work_sync(work: &rule->arfs_work); |
476 | if (rule->rule) |
477 | mlx5_del_flow_rules(fr: rule->rule); |
478 | hlist_del(n: &rule->hlist); |
479 | kfree(objp: rule); |
480 | } |
481 | } |
482 | |
483 | static struct hlist_head * |
484 | arfs_hash_bucket(struct arfs_table *arfs_t, __be16 src_port, |
485 | __be16 dst_port) |
486 | { |
487 | unsigned long l; |
488 | int bucket_idx; |
489 | |
490 | l = (__force unsigned long)src_port | |
491 | ((__force unsigned long)dst_port << 2); |
492 | |
493 | bucket_idx = hash_long(l, ARFS_HASH_SHIFT); |
494 | |
495 | return &arfs_t->rules_hash[bucket_idx]; |
496 | } |
497 | |
498 | static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, |
499 | u8 ip_proto, __be16 etype) |
500 | { |
501 | if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_TCP) |
502 | return &arfs->arfs_tables[ARFS_IPV4_TCP]; |
503 | if (etype == htons(ETH_P_IP) && ip_proto == IPPROTO_UDP) |
504 | return &arfs->arfs_tables[ARFS_IPV4_UDP]; |
505 | if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_TCP) |
506 | return &arfs->arfs_tables[ARFS_IPV6_TCP]; |
507 | if (etype == htons(ETH_P_IPV6) && ip_proto == IPPROTO_UDP) |
508 | return &arfs->arfs_tables[ARFS_IPV6_UDP]; |
509 | |
510 | return NULL; |
511 | } |
512 | |
513 | static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv, |
514 | struct arfs_rule *arfs_rule) |
515 | { |
516 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs: priv->fs); |
517 | struct arfs_tuple *tuple = &arfs_rule->tuple; |
518 | struct mlx5_flow_handle *rule = NULL; |
519 | struct mlx5_flow_destination dest = {}; |
520 | MLX5_DECLARE_FLOW_ACT(flow_act); |
521 | struct arfs_table *arfs_table; |
522 | struct mlx5_flow_spec *spec; |
523 | struct mlx5_flow_table *ft; |
524 | int err = 0; |
525 | |
526 | spec = kvzalloc(size: sizeof(*spec), GFP_KERNEL); |
527 | if (!spec) { |
528 | priv->channel_stats[arfs_rule->rxq]->rq.arfs_err++; |
529 | err = -ENOMEM; |
530 | goto out; |
531 | } |
532 | spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; |
533 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
534 | outer_headers.ethertype); |
535 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, |
536 | ntohs(tuple->etype)); |
537 | arfs_table = arfs_get_table(arfs, ip_proto: tuple->ip_proto, etype: tuple->etype); |
538 | if (!arfs_table) { |
539 | WARN_ONCE(1, "arfs table does not exist for etype %u and ip_proto %u\n" , |
540 | tuple->etype, tuple->ip_proto); |
541 | err = -EINVAL; |
542 | goto out; |
543 | } |
544 | |
545 | ft = arfs_table->ft.t; |
546 | if (tuple->ip_proto == IPPROTO_TCP) { |
547 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
548 | outer_headers.tcp_dport); |
549 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
550 | outer_headers.tcp_sport); |
551 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport, |
552 | ntohs(tuple->dst_port)); |
553 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport, |
554 | ntohs(tuple->src_port)); |
555 | } else { |
556 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
557 | outer_headers.udp_dport); |
558 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
559 | outer_headers.udp_sport); |
560 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, |
561 | ntohs(tuple->dst_port)); |
562 | MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_sport, |
563 | ntohs(tuple->src_port)); |
564 | } |
565 | if (tuple->etype == htons(ETH_P_IP)) { |
566 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
567 | outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), |
568 | &tuple->src_ipv4, |
569 | 4); |
570 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
571 | outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), |
572 | &tuple->dst_ipv4, |
573 | 4); |
574 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
575 | outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); |
576 | MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, |
577 | outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); |
578 | } else { |
579 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
580 | outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), |
581 | &tuple->src_ipv6, |
582 | 16); |
583 | memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, |
584 | outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), |
585 | &tuple->dst_ipv6, |
586 | 16); |
587 | memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, |
588 | outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), |
589 | 0xff, |
590 | 16); |
591 | memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, |
592 | outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), |
593 | 0xff, |
594 | 16); |
595 | } |
596 | dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
597 | dest.tir_num = mlx5e_rx_res_get_tirn_direct(res: priv->rx_res, ix: arfs_rule->rxq); |
598 | rule = mlx5_add_flow_rules(ft, spec, flow_act: &flow_act, dest: &dest, num_dest: 1); |
599 | if (IS_ERR(ptr: rule)) { |
600 | err = PTR_ERR(ptr: rule); |
601 | priv->channel_stats[arfs_rule->rxq]->rq.arfs_err++; |
602 | netdev_dbg(priv->netdev, |
603 | "%s: add rule(filter id=%d, rq idx=%d, ip proto=0x%x) failed,err=%d\n" , |
604 | __func__, arfs_rule->filter_id, arfs_rule->rxq, |
605 | tuple->ip_proto, err); |
606 | } |
607 | |
608 | out: |
609 | kvfree(addr: spec); |
610 | return err ? ERR_PTR(error: err) : rule; |
611 | } |
612 | |
613 | static void arfs_modify_rule_rq(struct mlx5e_priv *priv, |
614 | struct mlx5_flow_handle *rule, u16 rxq) |
615 | { |
616 | struct mlx5_flow_destination dst = {}; |
617 | int err = 0; |
618 | |
619 | dst.type = MLX5_FLOW_DESTINATION_TYPE_TIR; |
620 | dst.tir_num = mlx5e_rx_res_get_tirn_direct(res: priv->rx_res, ix: rxq); |
621 | err = mlx5_modify_rule_destination(handler: rule, new_dest: &dst, NULL); |
622 | if (err) { |
623 | priv->channel_stats[rxq]->rq.arfs_err++; |
624 | netdev_warn(dev: priv->netdev, |
625 | format: "Failed to modify aRFS rule destination to rq=%d\n" , rxq); |
626 | } |
627 | } |
628 | |
629 | static void arfs_handle_work(struct work_struct *work) |
630 | { |
631 | struct arfs_rule *arfs_rule = container_of(work, |
632 | struct arfs_rule, |
633 | arfs_work); |
634 | struct mlx5e_priv *priv = arfs_rule->priv; |
635 | struct mlx5e_arfs_tables *arfs; |
636 | struct mlx5_flow_handle *rule; |
637 | |
638 | arfs = mlx5e_fs_get_arfs(fs: priv->fs); |
639 | if (!test_bit(MLX5E_ARFS_STATE_ENABLED, &arfs->state)) |
640 | return; |
641 | |
642 | if (!arfs_rule->rule) { |
643 | rule = arfs_add_rule(priv, arfs_rule); |
644 | if (IS_ERR(ptr: rule)) |
645 | goto out; |
646 | arfs_rule->rule = rule; |
647 | priv->channel_stats[arfs_rule->rxq]->rq.arfs_add++; |
648 | } else { |
649 | arfs_modify_rule_rq(priv, rule: arfs_rule->rule, |
650 | rxq: arfs_rule->rxq); |
651 | } |
652 | out: |
653 | arfs_may_expire_flow(priv); |
654 | } |
655 | |
656 | static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, |
657 | struct arfs_table *arfs_t, |
658 | const struct flow_keys *fk, |
659 | u16 rxq, u32 flow_id) |
660 | { |
661 | struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs: priv->fs); |
662 | struct arfs_rule *rule; |
663 | struct arfs_tuple *tuple; |
664 | |
665 | rule = kzalloc(size: sizeof(*rule), GFP_ATOMIC); |
666 | if (!rule) { |
667 | priv->channel_stats[rxq]->rq.arfs_err++; |
668 | return NULL; |
669 | } |
670 | |
671 | rule->priv = priv; |
672 | rule->rxq = rxq; |
673 | INIT_WORK(&rule->arfs_work, arfs_handle_work); |
674 | |
675 | tuple = &rule->tuple; |
676 | tuple->etype = fk->basic.n_proto; |
677 | tuple->ip_proto = fk->basic.ip_proto; |
678 | if (tuple->etype == htons(ETH_P_IP)) { |
679 | tuple->src_ipv4 = fk->addrs.v4addrs.src; |
680 | tuple->dst_ipv4 = fk->addrs.v4addrs.dst; |
681 | } else { |
682 | memcpy(&tuple->src_ipv6, &fk->addrs.v6addrs.src, |
683 | sizeof(struct in6_addr)); |
684 | memcpy(&tuple->dst_ipv6, &fk->addrs.v6addrs.dst, |
685 | sizeof(struct in6_addr)); |
686 | } |
687 | tuple->src_port = fk->ports.src; |
688 | tuple->dst_port = fk->ports.dst; |
689 | |
690 | rule->flow_id = flow_id; |
691 | rule->filter_id = arfs->last_filter_id++ % RPS_NO_FILTER; |
692 | |
693 | hlist_add_head(n: &rule->hlist, |
694 | h: arfs_hash_bucket(arfs_t, src_port: tuple->src_port, |
695 | dst_port: tuple->dst_port)); |
696 | return rule; |
697 | } |
698 | |
699 | static bool arfs_cmp(const struct arfs_tuple *tuple, const struct flow_keys *fk) |
700 | { |
701 | if (tuple->src_port != fk->ports.src || tuple->dst_port != fk->ports.dst) |
702 | return false; |
703 | if (tuple->etype != fk->basic.n_proto) |
704 | return false; |
705 | if (tuple->etype == htons(ETH_P_IP)) |
706 | return tuple->src_ipv4 == fk->addrs.v4addrs.src && |
707 | tuple->dst_ipv4 == fk->addrs.v4addrs.dst; |
708 | if (tuple->etype == htons(ETH_P_IPV6)) |
709 | return !memcmp(p: &tuple->src_ipv6, q: &fk->addrs.v6addrs.src, |
710 | size: sizeof(struct in6_addr)) && |
711 | !memcmp(p: &tuple->dst_ipv6, q: &fk->addrs.v6addrs.dst, |
712 | size: sizeof(struct in6_addr)); |
713 | return false; |
714 | } |
715 | |
716 | static struct arfs_rule *arfs_find_rule(struct arfs_table *arfs_t, |
717 | const struct flow_keys *fk) |
718 | { |
719 | struct arfs_rule *arfs_rule; |
720 | struct hlist_head *head; |
721 | |
722 | head = arfs_hash_bucket(arfs_t, src_port: fk->ports.src, dst_port: fk->ports.dst); |
723 | hlist_for_each_entry(arfs_rule, head, hlist) { |
724 | if (arfs_cmp(tuple: &arfs_rule->tuple, fk)) |
725 | return arfs_rule; |
726 | } |
727 | |
728 | return NULL; |
729 | } |
730 | |
731 | int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, |
732 | u16 rxq_index, u32 flow_id) |
733 | { |
734 | struct mlx5e_priv *priv = netdev_priv(dev); |
735 | struct mlx5e_arfs_tables *arfs; |
736 | struct arfs_rule *arfs_rule; |
737 | struct arfs_table *arfs_t; |
738 | struct flow_keys fk; |
739 | |
740 | arfs = mlx5e_fs_get_arfs(fs: priv->fs); |
741 | if (!skb_flow_dissect_flow_keys(skb, flow: &fk, flags: 0)) |
742 | return -EPROTONOSUPPORT; |
743 | |
744 | if (fk.basic.n_proto != htons(ETH_P_IP) && |
745 | fk.basic.n_proto != htons(ETH_P_IPV6)) |
746 | return -EPROTONOSUPPORT; |
747 | |
748 | if (skb->encapsulation) |
749 | return -EPROTONOSUPPORT; |
750 | |
751 | arfs_t = arfs_get_table(arfs, ip_proto: fk.basic.ip_proto, etype: fk.basic.n_proto); |
752 | if (!arfs_t) |
753 | return -EPROTONOSUPPORT; |
754 | |
755 | spin_lock_bh(lock: &arfs->arfs_lock); |
756 | if (!test_bit(MLX5E_ARFS_STATE_ENABLED, &arfs->state)) { |
757 | spin_unlock_bh(lock: &arfs->arfs_lock); |
758 | return -EPERM; |
759 | } |
760 | |
761 | arfs_rule = arfs_find_rule(arfs_t, fk: &fk); |
762 | if (arfs_rule) { |
763 | if (arfs_rule->rxq == rxq_index || work_busy(work: &arfs_rule->arfs_work)) { |
764 | spin_unlock_bh(lock: &arfs->arfs_lock); |
765 | return arfs_rule->filter_id; |
766 | } |
767 | |
768 | priv->channel_stats[rxq_index]->rq.arfs_request_in++; |
769 | priv->channel_stats[arfs_rule->rxq]->rq.arfs_request_out++; |
770 | arfs_rule->rxq = rxq_index; |
771 | } else { |
772 | arfs_rule = arfs_alloc_rule(priv, arfs_t, fk: &fk, rxq: rxq_index, flow_id); |
773 | if (!arfs_rule) { |
774 | spin_unlock_bh(lock: &arfs->arfs_lock); |
775 | return -ENOMEM; |
776 | } |
777 | } |
778 | queue_work(wq: arfs->wq, work: &arfs_rule->arfs_work); |
779 | spin_unlock_bh(lock: &arfs->arfs_lock); |
780 | return arfs_rule->filter_id; |
781 | } |
782 | |
783 | |