1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | // Copyright (c) 2019 Mellanox Technologies. |
3 | |
4 | #include <linux/mlx5/fs.h> |
5 | #include "eswitch.h" |
6 | #include "en_tc.h" |
7 | #include "fs_core.h" |
8 | |
9 | struct mlx5_termtbl_handle { |
10 | struct hlist_node termtbl_hlist; |
11 | |
12 | struct mlx5_flow_table *termtbl; |
13 | struct mlx5_flow_act flow_act; |
14 | struct mlx5_flow_destination dest; |
15 | |
16 | struct mlx5_flow_handle *rule; |
17 | int ref_count; |
18 | }; |
19 | |
20 | static u32 |
21 | mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act, |
22 | struct mlx5_flow_destination *dest) |
23 | { |
24 | u32 hash; |
25 | |
26 | hash = jhash_1word(a: flow_act->action, initval: 0); |
27 | hash = jhash(key: (const void *)&flow_act->vlan, |
28 | length: sizeof(flow_act->vlan), initval: hash); |
29 | hash = jhash(key: (const void *)&dest->vport.num, |
30 | length: sizeof(dest->vport.num), initval: hash); |
31 | hash = jhash(key: (const void *)&dest->vport.vhca_id, |
32 | length: sizeof(dest->vport.num), initval: hash); |
33 | if (flow_act->pkt_reformat) |
34 | hash = jhash(key: flow_act->pkt_reformat, |
35 | length: sizeof(*flow_act->pkt_reformat), |
36 | initval: hash); |
37 | return hash; |
38 | } |
39 | |
40 | static int |
41 | mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1, |
42 | struct mlx5_flow_destination *dest1, |
43 | struct mlx5_flow_act *flow_act2, |
44 | struct mlx5_flow_destination *dest2) |
45 | { |
46 | int ret; |
47 | |
48 | ret = flow_act1->action != flow_act2->action || |
49 | dest1->vport.num != dest2->vport.num || |
50 | dest1->vport.vhca_id != dest2->vport.vhca_id || |
51 | memcmp(p: &flow_act1->vlan, q: &flow_act2->vlan, |
52 | size: sizeof(flow_act1->vlan)); |
53 | if (ret) |
54 | return ret; |
55 | |
56 | if (flow_act1->pkt_reformat && flow_act2->pkt_reformat) |
57 | return memcmp(p: flow_act1->pkt_reformat, q: flow_act2->pkt_reformat, |
58 | size: sizeof(*flow_act1->pkt_reformat)); |
59 | |
60 | return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat); |
61 | } |
62 | |
63 | static int |
64 | mlx5_eswitch_termtbl_create(struct mlx5_core_dev *dev, |
65 | struct mlx5_termtbl_handle *tt, |
66 | struct mlx5_flow_act *flow_act) |
67 | { |
68 | struct mlx5_flow_table_attr ft_attr = {}; |
69 | struct mlx5_flow_namespace *root_ns; |
70 | int err, err2; |
71 | |
72 | root_ns = mlx5_get_flow_namespace(dev, type: MLX5_FLOW_NAMESPACE_FDB); |
73 | if (!root_ns) { |
74 | esw_warn(dev, "Failed to get FDB flow namespace\n" ); |
75 | return -EOPNOTSUPP; |
76 | } |
77 | |
78 | /* As this is the terminating action then the termination table is the |
79 | * same prio as the slow path |
80 | */ |
81 | ft_attr.flags = MLX5_FLOW_TABLE_TERMINATION | MLX5_FLOW_TABLE_UNMANAGED | |
82 | MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; |
83 | ft_attr.prio = FDB_TC_OFFLOAD; |
84 | ft_attr.max_fte = 1; |
85 | ft_attr.level = 1; |
86 | ft_attr.autogroup.max_num_groups = 1; |
87 | tt->termtbl = mlx5_create_auto_grouped_flow_table(ns: root_ns, ft_attr: &ft_attr); |
88 | if (IS_ERR(ptr: tt->termtbl)) { |
89 | err = PTR_ERR(ptr: tt->termtbl); |
90 | esw_warn(dev, "Failed to create termination table, err %pe\n" , tt->termtbl); |
91 | return err; |
92 | } |
93 | |
94 | tt->rule = mlx5_add_flow_rules(ft: tt->termtbl, NULL, flow_act, |
95 | dest: &tt->dest, num_dest: 1); |
96 | if (IS_ERR(ptr: tt->rule)) { |
97 | err = PTR_ERR(ptr: tt->rule); |
98 | esw_warn(dev, "Failed to create termination table rule, err %pe\n" , tt->rule); |
99 | goto add_flow_err; |
100 | } |
101 | return 0; |
102 | |
103 | add_flow_err: |
104 | err2 = mlx5_destroy_flow_table(ft: tt->termtbl); |
105 | if (err2) |
106 | esw_warn(dev, "Failed to destroy termination table, err %d\n" , err2); |
107 | |
108 | return err; |
109 | } |
110 | |
111 | static struct mlx5_termtbl_handle * |
112 | mlx5_eswitch_termtbl_get_create(struct mlx5_eswitch *esw, |
113 | struct mlx5_flow_act *flow_act, |
114 | struct mlx5_flow_destination *dest, |
115 | struct mlx5_esw_flow_attr *attr) |
116 | { |
117 | struct mlx5_termtbl_handle *tt; |
118 | bool found = false; |
119 | u32 hash_key; |
120 | int err; |
121 | |
122 | mutex_lock(&esw->offloads.termtbl_mutex); |
123 | hash_key = mlx5_eswitch_termtbl_hash(flow_act, dest); |
124 | hash_for_each_possible(esw->offloads.termtbl_tbl, tt, |
125 | termtbl_hlist, hash_key) { |
126 | if (!mlx5_eswitch_termtbl_cmp(flow_act1: &tt->flow_act, dest1: &tt->dest, |
127 | flow_act2: flow_act, dest2: dest)) { |
128 | found = true; |
129 | break; |
130 | } |
131 | } |
132 | if (found) |
133 | goto tt_add_ref; |
134 | |
135 | tt = kzalloc(size: sizeof(*tt), GFP_KERNEL); |
136 | if (!tt) { |
137 | err = -ENOMEM; |
138 | goto tt_create_err; |
139 | } |
140 | |
141 | tt->dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; |
142 | tt->dest.vport.num = dest->vport.num; |
143 | tt->dest.vport.vhca_id = dest->vport.vhca_id; |
144 | tt->dest.vport.flags = dest->vport.flags; |
145 | memcpy(&tt->flow_act, flow_act, sizeof(*flow_act)); |
146 | |
147 | err = mlx5_eswitch_termtbl_create(dev: esw->dev, tt, flow_act); |
148 | if (err) |
149 | goto tt_create_err; |
150 | |
151 | hash_add(esw->offloads.termtbl_tbl, &tt->termtbl_hlist, hash_key); |
152 | tt_add_ref: |
153 | tt->ref_count++; |
154 | mutex_unlock(lock: &esw->offloads.termtbl_mutex); |
155 | return tt; |
156 | tt_create_err: |
157 | kfree(objp: tt); |
158 | mutex_unlock(lock: &esw->offloads.termtbl_mutex); |
159 | return ERR_PTR(error: err); |
160 | } |
161 | |
162 | void |
163 | mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw, |
164 | struct mlx5_termtbl_handle *tt) |
165 | { |
166 | mutex_lock(&esw->offloads.termtbl_mutex); |
167 | if (--tt->ref_count == 0) |
168 | hash_del(node: &tt->termtbl_hlist); |
169 | mutex_unlock(lock: &esw->offloads.termtbl_mutex); |
170 | |
171 | if (!tt->ref_count) { |
172 | mlx5_del_flow_rules(fr: tt->rule); |
173 | mlx5_destroy_flow_table(ft: tt->termtbl); |
174 | kfree(objp: tt); |
175 | } |
176 | } |
177 | |
178 | static void |
179 | mlx5_eswitch_termtbl_actions_move(struct mlx5_flow_act *src, |
180 | struct mlx5_flow_act *dst) |
181 | { |
182 | if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { |
183 | src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; |
184 | dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH; |
185 | memcpy(&dst->vlan[0], &src->vlan[0], sizeof(src->vlan[0])); |
186 | memset(&src->vlan[0], 0, sizeof(src->vlan[0])); |
187 | |
188 | if (src->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { |
189 | src->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; |
190 | dst->action |= MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2; |
191 | memcpy(&dst->vlan[1], &src->vlan[1], sizeof(src->vlan[1])); |
192 | memset(&src->vlan[1], 0, sizeof(src->vlan[1])); |
193 | } |
194 | } |
195 | } |
196 | |
197 | static bool mlx5_eswitch_offload_is_uplink_port(const struct mlx5_eswitch *esw, |
198 | const struct mlx5_flow_spec *spec) |
199 | { |
200 | u16 port_mask, port_value; |
201 | |
202 | if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) |
203 | return spec->flow_context.flow_source == |
204 | MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; |
205 | |
206 | port_mask = MLX5_GET(fte_match_param, spec->match_criteria, |
207 | misc_parameters.source_port); |
208 | port_value = MLX5_GET(fte_match_param, spec->match_value, |
209 | misc_parameters.source_port); |
210 | return (port_mask & port_value) == MLX5_VPORT_UPLINK; |
211 | } |
212 | |
213 | bool |
214 | mlx5_eswitch_termtbl_required(struct mlx5_eswitch *esw, |
215 | struct mlx5_flow_attr *attr, |
216 | struct mlx5_flow_act *flow_act, |
217 | struct mlx5_flow_spec *spec) |
218 | { |
219 | struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; |
220 | int i; |
221 | |
222 | if (!MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, termination_table) || |
223 | !MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level) || |
224 | mlx5e_tc_attr_flags_skip(attr_flags: attr->flags) || |
225 | (!mlx5_eswitch_offload_is_uplink_port(esw, spec) && !esw_attr->int_port)) |
226 | return false; |
227 | |
228 | /* push vlan on RX */ |
229 | if (flow_act->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH && |
230 | !(mlx5_fs_get_capabilities(dev: esw->dev, type: MLX5_FLOW_NAMESPACE_FDB) & |
231 | MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX)) |
232 | return true; |
233 | |
234 | /* hairpin */ |
235 | for (i = esw_attr->split_count; i < esw_attr->out_count; i++) |
236 | if (!esw_attr->dest_int_port && esw_attr->dests[i].vport_valid && |
237 | esw_attr->dests[i].vport == MLX5_VPORT_UPLINK) |
238 | return true; |
239 | |
240 | return false; |
241 | } |
242 | |
243 | struct mlx5_flow_handle * |
244 | mlx5_eswitch_add_termtbl_rule(struct mlx5_eswitch *esw, |
245 | struct mlx5_flow_table *fdb, |
246 | struct mlx5_flow_spec *spec, |
247 | struct mlx5_esw_flow_attr *attr, |
248 | struct mlx5_flow_act *flow_act, |
249 | struct mlx5_flow_destination *dest, |
250 | int num_dest) |
251 | { |
252 | struct mlx5_flow_act term_tbl_act = {}; |
253 | struct mlx5_flow_handle *rule = NULL; |
254 | bool term_table_created = false; |
255 | int num_vport_dests = 0; |
256 | int i, curr_dest; |
257 | |
258 | mlx5_eswitch_termtbl_actions_move(src: flow_act, dst: &term_tbl_act); |
259 | term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; |
260 | |
261 | for (i = 0; i < num_dest; i++) { |
262 | struct mlx5_termtbl_handle *tt; |
263 | |
264 | /* only vport destinations can be terminated */ |
265 | if (dest[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT) |
266 | continue; |
267 | |
268 | if (attr->dests[num_vport_dests].flags & MLX5_ESW_DEST_ENCAP) { |
269 | term_tbl_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; |
270 | term_tbl_act.pkt_reformat = attr->dests[num_vport_dests].pkt_reformat; |
271 | } else { |
272 | term_tbl_act.action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; |
273 | term_tbl_act.pkt_reformat = NULL; |
274 | } |
275 | |
276 | /* get the terminating table for the action list */ |
277 | tt = mlx5_eswitch_termtbl_get_create(esw, flow_act: &term_tbl_act, |
278 | dest: &dest[i], attr); |
279 | if (IS_ERR(ptr: tt)) { |
280 | esw_warn(esw->dev, "Failed to get termination table, err %pe\n" , tt); |
281 | goto revert_changes; |
282 | } |
283 | attr->dests[num_vport_dests].termtbl = tt; |
284 | num_vport_dests++; |
285 | |
286 | /* link the destination with the termination table */ |
287 | dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; |
288 | dest[i].ft = tt->termtbl; |
289 | term_table_created = true; |
290 | } |
291 | |
292 | /* at least one destination should reference a termination table */ |
293 | if (!term_table_created) |
294 | goto revert_changes; |
295 | |
296 | /* create the FTE */ |
297 | flow_act->action &= ~MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; |
298 | flow_act->pkt_reformat = NULL; |
299 | flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; |
300 | rule = mlx5_add_flow_rules(ft: fdb, spec, flow_act, dest, num_dest); |
301 | if (IS_ERR(ptr: rule)) |
302 | goto revert_changes; |
303 | |
304 | goto out; |
305 | |
306 | revert_changes: |
307 | /* revert the changes that were made to the original flow_act |
308 | * and fall-back to the original rule actions |
309 | */ |
310 | mlx5_eswitch_termtbl_actions_move(src: &term_tbl_act, dst: flow_act); |
311 | |
312 | for (curr_dest = 0; curr_dest < num_vport_dests; curr_dest++) { |
313 | struct mlx5_termtbl_handle *tt = attr->dests[curr_dest].termtbl; |
314 | |
315 | attr->dests[curr_dest].termtbl = NULL; |
316 | |
317 | /* search for the destination associated with the |
318 | * current term table |
319 | */ |
320 | for (i = 0; i < num_dest; i++) { |
321 | if (dest[i].ft != tt->termtbl) |
322 | continue; |
323 | |
324 | memset(&dest[i], 0, sizeof(dest[i])); |
325 | dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; |
326 | dest[i].vport.num = tt->dest.vport.num; |
327 | dest[i].vport.vhca_id = tt->dest.vport.vhca_id; |
328 | mlx5_eswitch_termtbl_put(esw, tt); |
329 | break; |
330 | } |
331 | } |
332 | rule = mlx5_add_flow_rules(ft: fdb, spec, flow_act, dest, num_dest); |
333 | out: |
334 | return rule; |
335 | } |
336 | |