1 | /* |
2 | * Copyright (c) 2015, Mellanox Technologies. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | */ |
32 | |
33 | #ifndef _MLX5_FS_CORE_ |
34 | #define _MLX5_FS_CORE_ |
35 | |
36 | #include <linux/refcount.h> |
37 | #include <linux/mlx5/fs.h> |
38 | #include <linux/rhashtable.h> |
39 | #include <linux/llist.h> |
40 | #include <steering/fs_dr.h> |
41 | |
42 | #define FDB_TC_MAX_CHAIN 3 |
43 | #define FDB_FT_CHAIN (FDB_TC_MAX_CHAIN + 1) |
44 | #define FDB_TC_SLOW_PATH_CHAIN (FDB_FT_CHAIN + 1) |
45 | |
46 | /* The index of the last real chain (FT) + 1 as chain zero is valid as well */ |
47 | #define FDB_NUM_CHAINS (FDB_FT_CHAIN + 1) |
48 | |
49 | #define FDB_TC_MAX_PRIO 16 |
50 | #define FDB_TC_LEVELS_PER_PRIO 2 |
51 | |
52 | struct mlx5_flow_definer { |
53 | enum mlx5_flow_namespace_type ns_type; |
54 | u32 id; |
55 | }; |
56 | |
57 | enum mlx5_flow_resource_owner { |
58 | MLX5_FLOW_RESOURCE_OWNER_FW, |
59 | MLX5_FLOW_RESOURCE_OWNER_SW, |
60 | }; |
61 | |
62 | struct mlx5_modify_hdr { |
63 | enum mlx5_flow_namespace_type ns_type; |
64 | enum mlx5_flow_resource_owner owner; |
65 | union { |
66 | struct mlx5_fs_dr_action action; |
67 | u32 id; |
68 | }; |
69 | }; |
70 | |
71 | struct mlx5_pkt_reformat { |
72 | enum mlx5_flow_namespace_type ns_type; |
73 | int reformat_type; /* from mlx5_ifc */ |
74 | enum mlx5_flow_resource_owner owner; |
75 | union { |
76 | struct mlx5_fs_dr_action action; |
77 | u32 id; |
78 | }; |
79 | }; |
80 | |
81 | /* FS_TYPE_PRIO_CHAINS is a PRIO that will have namespaces only, |
82 | * and those are in parallel to one another when going over them to connect |
83 | * a new flow table. Meaning the last flow table in a TYPE_PRIO prio in one |
84 | * parallel namespace will not automatically connect to the first flow table |
85 | * found in any prio in any next namespace, but skip the entire containing |
86 | * TYPE_PRIO_CHAINS prio. |
87 | * |
88 | * This is used to implement tc chains, each chain of prios is a different |
89 | * namespace inside a containing TYPE_PRIO_CHAINS prio. |
90 | */ |
91 | |
92 | enum fs_node_type { |
93 | FS_TYPE_NAMESPACE, |
94 | FS_TYPE_PRIO, |
95 | FS_TYPE_PRIO_CHAINS, |
96 | FS_TYPE_FLOW_TABLE, |
97 | FS_TYPE_FLOW_GROUP, |
98 | FS_TYPE_FLOW_ENTRY, |
99 | FS_TYPE_FLOW_DEST |
100 | }; |
101 | |
102 | enum fs_flow_table_type { |
103 | FS_FT_NIC_RX = 0x0, |
104 | FS_FT_NIC_TX = 0x1, |
105 | FS_FT_ESW_EGRESS_ACL = 0x2, |
106 | FS_FT_ESW_INGRESS_ACL = 0x3, |
107 | FS_FT_FDB = 0X4, |
108 | FS_FT_SNIFFER_RX = 0X5, |
109 | FS_FT_SNIFFER_TX = 0X6, |
110 | FS_FT_RDMA_RX = 0X7, |
111 | FS_FT_RDMA_TX = 0X8, |
112 | FS_FT_PORT_SEL = 0X9, |
113 | FS_FT_MAX_TYPE = FS_FT_PORT_SEL, |
114 | }; |
115 | |
116 | enum fs_flow_table_op_mod { |
117 | FS_FT_OP_MOD_NORMAL, |
118 | FS_FT_OP_MOD_LAG_DEMUX, |
119 | }; |
120 | |
121 | enum fs_fte_status { |
122 | FS_FTE_STATUS_EXISTING = 1UL << 0, |
123 | }; |
124 | |
125 | enum mlx5_flow_steering_mode { |
126 | MLX5_FLOW_STEERING_MODE_DMFS, |
127 | MLX5_FLOW_STEERING_MODE_SMFS |
128 | }; |
129 | |
130 | enum mlx5_flow_steering_capabilty { |
131 | MLX5_FLOW_STEERING_CAP_VLAN_PUSH_ON_RX = 1UL << 0, |
132 | MLX5_FLOW_STEERING_CAP_VLAN_POP_ON_TX = 1UL << 1, |
133 | MLX5_FLOW_STEERING_CAP_MATCH_RANGES = 1UL << 2, |
134 | }; |
135 | |
136 | struct mlx5_flow_steering { |
137 | struct mlx5_core_dev *dev; |
138 | enum mlx5_flow_steering_mode mode; |
139 | struct kmem_cache *fgs_cache; |
140 | struct kmem_cache *ftes_cache; |
141 | struct mlx5_flow_root_namespace *root_ns; |
142 | struct mlx5_flow_root_namespace *fdb_root_ns; |
143 | struct mlx5_flow_namespace **fdb_sub_ns; |
144 | struct mlx5_flow_root_namespace **esw_egress_root_ns; |
145 | struct mlx5_flow_root_namespace **esw_ingress_root_ns; |
146 | struct mlx5_flow_root_namespace *sniffer_tx_root_ns; |
147 | struct mlx5_flow_root_namespace *sniffer_rx_root_ns; |
148 | struct mlx5_flow_root_namespace *rdma_rx_root_ns; |
149 | struct mlx5_flow_root_namespace *rdma_tx_root_ns; |
150 | struct mlx5_flow_root_namespace *egress_root_ns; |
151 | struct mlx5_flow_root_namespace *port_sel_root_ns; |
152 | int esw_egress_acl_vports; |
153 | int esw_ingress_acl_vports; |
154 | }; |
155 | |
156 | struct fs_node { |
157 | struct list_head list; |
158 | struct list_head children; |
159 | enum fs_node_type type; |
160 | struct fs_node *parent; |
161 | struct fs_node *root; |
162 | /* lock the node for writing and traversing */ |
163 | struct rw_semaphore lock; |
164 | refcount_t refcount; |
165 | bool active; |
166 | void (*del_hw_func)(struct fs_node *); |
167 | void (*del_sw_func)(struct fs_node *); |
168 | atomic_t version; |
169 | }; |
170 | |
171 | struct mlx5_flow_rule { |
172 | struct fs_node node; |
173 | struct mlx5_flow_table *ft; |
174 | struct mlx5_flow_destination dest_attr; |
175 | /* next_ft should be accessed under chain_lock and only of |
176 | * destination type is FWD_NEXT_fT. |
177 | */ |
178 | struct list_head next_ft; |
179 | u32 sw_action; |
180 | }; |
181 | |
182 | struct mlx5_flow_handle { |
183 | int num_rules; |
184 | struct mlx5_flow_rule *rule[] __counted_by(num_rules); |
185 | }; |
186 | |
187 | /* Type of children is mlx5_flow_group */ |
188 | struct mlx5_flow_table { |
189 | struct fs_node node; |
190 | struct mlx5_fs_dr_table fs_dr_table; |
191 | u32 id; |
192 | u16 vport; |
193 | unsigned int max_fte; |
194 | unsigned int level; |
195 | enum fs_flow_table_type type; |
196 | enum fs_flow_table_op_mod op_mod; |
197 | struct { |
198 | bool active; |
199 | unsigned int required_groups; |
200 | unsigned int group_size; |
201 | unsigned int num_groups; |
202 | unsigned int max_fte; |
203 | } autogroup; |
204 | /* Protect fwd_rules */ |
205 | struct mutex lock; |
206 | /* FWD rules that point on this flow table */ |
207 | struct list_head fwd_rules; |
208 | u32 flags; |
209 | struct rhltable fgs_hash; |
210 | enum mlx5_flow_table_miss_action def_miss_action; |
211 | struct mlx5_flow_namespace *ns; |
212 | }; |
213 | |
214 | struct mlx5_ft_underlay_qp { |
215 | struct list_head list; |
216 | u32 qpn; |
217 | }; |
218 | |
219 | #define MLX5_FTE_MATCH_PARAM_RESERVED reserved_at_e00 |
220 | /* Calculate the fte_match_param length and without the reserved length. |
221 | * Make sure the reserved field is the last. |
222 | */ |
223 | #define MLX5_ST_SZ_DW_MATCH_PARAM \ |
224 | ((MLX5_BYTE_OFF(fte_match_param, MLX5_FTE_MATCH_PARAM_RESERVED) / sizeof(u32)) + \ |
225 | BUILD_BUG_ON_ZERO(MLX5_ST_SZ_BYTES(fte_match_param) != \ |
226 | MLX5_FLD_SZ_BYTES(fte_match_param, \ |
227 | MLX5_FTE_MATCH_PARAM_RESERVED) +\ |
228 | MLX5_BYTE_OFF(fte_match_param, \ |
229 | MLX5_FTE_MATCH_PARAM_RESERVED))) |
230 | |
231 | /* Type of children is mlx5_flow_rule */ |
232 | struct fs_fte { |
233 | struct fs_node node; |
234 | struct mlx5_fs_dr_rule fs_dr_rule; |
235 | u32 val[MLX5_ST_SZ_DW_MATCH_PARAM]; |
236 | u32 dests_size; |
237 | u32 fwd_dests; |
238 | u32 index; |
239 | struct mlx5_flow_context flow_context; |
240 | struct mlx5_flow_act action; |
241 | enum fs_fte_status status; |
242 | struct mlx5_fc *counter; |
243 | struct rhash_head hash; |
244 | int modify_mask; |
245 | }; |
246 | |
247 | /* Type of children is mlx5_flow_table/namespace */ |
248 | struct fs_prio { |
249 | struct fs_node node; |
250 | unsigned int num_levels; |
251 | unsigned int start_level; |
252 | unsigned int prio; |
253 | unsigned int num_ft; |
254 | }; |
255 | |
256 | /* Type of children is fs_prio */ |
257 | struct mlx5_flow_namespace { |
258 | /* parent == NULL => root ns */ |
259 | struct fs_node node; |
260 | enum mlx5_flow_table_miss_action def_miss_action; |
261 | }; |
262 | |
263 | struct mlx5_flow_group_mask { |
264 | u8 match_criteria_enable; |
265 | u32 match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM]; |
266 | }; |
267 | |
268 | /* Type of children is fs_fte */ |
269 | struct mlx5_flow_group { |
270 | struct fs_node node; |
271 | struct mlx5_fs_dr_matcher fs_dr_matcher; |
272 | struct mlx5_flow_group_mask mask; |
273 | u32 start_index; |
274 | u32 max_ftes; |
275 | struct ida fte_allocator; |
276 | u32 id; |
277 | struct rhashtable ftes_hash; |
278 | struct rhlist_head hash; |
279 | }; |
280 | |
281 | struct mlx5_flow_root_namespace { |
282 | struct mlx5_flow_namespace ns; |
283 | enum mlx5_flow_steering_mode mode; |
284 | struct mlx5_fs_dr_domain fs_dr_domain; |
285 | enum fs_flow_table_type table_type; |
286 | struct mlx5_core_dev *dev; |
287 | struct mlx5_flow_table *root_ft; |
288 | /* Should be held when chaining flow tables */ |
289 | struct mutex chain_lock; |
290 | struct list_head underlay_qpns; |
291 | const struct mlx5_flow_cmds *cmds; |
292 | }; |
293 | |
294 | int mlx5_init_fc_stats(struct mlx5_core_dev *dev); |
295 | void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev); |
296 | void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, |
297 | struct delayed_work *dwork, |
298 | unsigned long delay); |
299 | void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev, |
300 | unsigned long interval); |
301 | |
302 | const struct mlx5_flow_cmds *mlx5_fs_cmd_get_fw_cmds(void); |
303 | |
304 | int mlx5_flow_namespace_set_peer(struct mlx5_flow_root_namespace *ns, |
305 | struct mlx5_flow_root_namespace *peer_ns, |
306 | u16 peer_vhca_id); |
307 | |
308 | int mlx5_flow_namespace_set_mode(struct mlx5_flow_namespace *ns, |
309 | enum mlx5_flow_steering_mode mode); |
310 | |
311 | int mlx5_fs_core_alloc(struct mlx5_core_dev *dev); |
312 | void mlx5_fs_core_free(struct mlx5_core_dev *dev); |
313 | int mlx5_fs_core_init(struct mlx5_core_dev *dev); |
314 | void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev); |
315 | |
316 | int mlx5_fs_egress_acls_init(struct mlx5_core_dev *dev, int total_vports); |
317 | void mlx5_fs_egress_acls_cleanup(struct mlx5_core_dev *dev); |
318 | int mlx5_fs_ingress_acls_init(struct mlx5_core_dev *dev, int total_vports); |
319 | void mlx5_fs_ingress_acls_cleanup(struct mlx5_core_dev *dev); |
320 | |
321 | u32 mlx5_fs_get_capabilities(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type); |
322 | |
323 | struct mlx5_flow_root_namespace *find_root(struct fs_node *node); |
324 | |
325 | #define fs_get_obj(v, _node) {v = container_of((_node), typeof(*v), node); } |
326 | |
327 | #define fs_list_for_each_entry(pos, root) \ |
328 | list_for_each_entry(pos, root, node.list) |
329 | |
330 | #define fs_list_for_each_entry_safe(pos, tmp, root) \ |
331 | list_for_each_entry_safe(pos, tmp, root, node.list) |
332 | |
333 | #define fs_for_each_ns_or_ft_reverse(pos, prio) \ |
334 | list_for_each_entry_reverse(pos, &(prio)->node.children, list) |
335 | |
336 | #define fs_for_each_ns_or_ft(pos, prio) \ |
337 | list_for_each_entry(pos, (&(prio)->node.children), list) |
338 | |
339 | #define fs_for_each_prio(pos, ns) \ |
340 | fs_list_for_each_entry(pos, &(ns)->node.children) |
341 | |
342 | #define fs_for_each_ns(pos, prio) \ |
343 | fs_list_for_each_entry(pos, &(prio)->node.children) |
344 | |
345 | #define fs_for_each_ft(pos, prio) \ |
346 | fs_list_for_each_entry(pos, &(prio)->node.children) |
347 | |
348 | #define fs_for_each_ft_safe(pos, tmp, prio) \ |
349 | fs_list_for_each_entry_safe(pos, tmp, &(prio)->node.children) |
350 | |
351 | #define fs_for_each_fg(pos, ft) \ |
352 | fs_list_for_each_entry(pos, &(ft)->node.children) |
353 | |
354 | #define fs_for_each_fte(pos, fg) \ |
355 | fs_list_for_each_entry(pos, &(fg)->node.children) |
356 | |
357 | #define fs_for_each_dst(pos, fte) \ |
358 | fs_list_for_each_entry(pos, &(fte)->node.children) |
359 | |
360 | #define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \ |
361 | (type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \ |
362 | (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \ |
363 | (type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \ |
364 | (type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \ |
365 | (type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \ |
366 | (type == FS_FT_SNIFFER_RX) ? MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) : \ |
367 | (type == FS_FT_SNIFFER_TX) ? MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) : \ |
368 | (type == FS_FT_RDMA_RX) ? MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) : \ |
369 | (type == FS_FT_RDMA_TX) ? MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) : \ |
370 | (type == FS_FT_PORT_SEL) ? MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) : \ |
371 | (BUILD_BUG_ON_ZERO(FS_FT_PORT_SEL != FS_FT_MAX_TYPE))\ |
372 | ) |
373 | |
374 | #endif |
375 | |