1	/*
2	* Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3	*
4	* This software is available to you under a choice of one of two
5	* licenses. You may choose to be licensed under the terms of the GNU
6	* General Public License (GPL) Version 2, available from the file
7	* COPYING in the main directory of this source tree, or the
8	* OpenIB.org BSD license below:
9	*
10	* Redistribution and use in source and binary forms, with or
11	* without modification, are permitted provided that the following
12	* conditions are met:
13	*
14	* - Redistributions of source code must retain the above
15	* copyright notice, this list of conditions and the following
16	* disclaimer.
17	*
18	* - Redistributions in binary form must reproduce the above
19	* copyright notice, this list of conditions and the following
20	* disclaimer in the documentation and/or other materials
21	* provided with the distribution.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30	* SOFTWARE.
31	*/
32
33	#include <net/flow_dissector.h>
34	#include <net/flow_offload.h>
35	#include <net/sch_generic.h>
36	#include <net/pkt_cls.h>
37	#include <linux/mlx5/fs.h>
38	#include <linux/mlx5/device.h>
39	#include <linux/rhashtable.h>
40	#include <linux/refcount.h>
41	#include <linux/completion.h>
42	#include <net/arp.h>
43	#include <net/ipv6_stubs.h>
44	#include <net/bareudp.h>
45	#include <net/bonding.h>
46	#include <net/dst_metadata.h>
47	#include "devlink.h"
48	#include "en.h"
49	#include "en/tc/post_act.h"
50	#include "en/tc/act_stats.h"
51	#include "en_rep.h"
52	#include "en/rep/tc.h"
53	#include "en/rep/neigh.h"
54	#include "en_tc.h"
55	#include "eswitch.h"
56	#include "fs_core.h"
57	#include "en/port.h"
58	#include "en/tc_tun.h"
59	#include "en/mapping.h"
60	#include "en/tc_ct.h"
61	#include "en/mod_hdr.h"
62	#include "en/tc_tun_encap.h"
63	#include "en/tc/sample.h"
64	#include "en/tc/act/act.h"
65	#include "en/tc/post_meter.h"
66	#include "lib/devcom.h"
67	#include "lib/geneve.h"
68	#include "lib/fs_chains.h"
69	#include "diag/en_tc_tracepoint.h"
70	#include <asm/div64.h>
71	#include "lag/lag.h"
72	#include "lag/mp.h"
73
74	#define MLX5E_TC_TABLE_NUM_GROUPS 4
75	#define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(18)
76
77	struct mlx5e_tc_table {
78	/* Protects the dynamic assignment of the t parameter
79	* which is the nic tc root table.
80	*/
81	struct mutex t_lock;
82	struct mlx5e_priv *priv;
83	struct mlx5_flow_table *t;
84	struct mlx5_flow_table *miss_t;
85	struct mlx5_fs_chains *chains;
86	struct mlx5e_post_act *post_act;
87
88	struct rhashtable ht;
89
90	struct mod_hdr_tbl mod_hdr;
91	struct mutex hairpin_tbl_lock; /* protects hairpin_tbl */
92	DECLARE_HASHTABLE(hairpin_tbl, 8);
93
94	struct notifier_block netdevice_nb;
95	struct netdev_net_notifier netdevice_nn;
96
97	struct mlx5_tc_ct_priv *ct;
98	struct mapping_ctx *mapping;
99	struct dentry *dfs_root;
100
101	/* tc action stats */
102	struct mlx5e_tc_act_stats_handle *action_stats_handle;
103	};
104
105	struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = {
106	[MAPPED_OBJ_TO_REG] = {
107	.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0,
108	.moffset = 0,
109	.mlen = 16,
110	},
111	[VPORT_TO_REG] = {
112	.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0,
113	.moffset = 16,
114	.mlen = 16,
115	},
116	[TUNNEL_TO_REG] = {
117	.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,
118	.moffset = 8,
119	.mlen = ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS,
120	.soffset = MLX5_BYTE_OFF(fte_match_param,
121	misc_parameters_2.metadata_reg_c_1),
122	},
123	[ZONE_TO_REG] = zone_to_reg_ct,
124	[ZONE_RESTORE_TO_REG] = zone_restore_to_reg_ct,
125	[CTSTATE_TO_REG] = ctstate_to_reg_ct,
126	[MARK_TO_REG] = mark_to_reg_ct,
127	[LABELS_TO_REG] = labels_to_reg_ct,
128	[FTEID_TO_REG] = fteid_to_reg_ct,
129	/* For NIC rules we store the restore metadata directly
130	* into reg_b that is passed to SW since we don't
131	* jump between steering domains.
132	*/
133	[NIC_MAPPED_OBJ_TO_REG] = {
134	.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_B,
135	.moffset = 0,
136	.mlen = 16,
137	},
138	[NIC_ZONE_RESTORE_TO_REG] = nic_zone_restore_to_reg_ct,
139	[PACKET_COLOR_TO_REG] = packet_color_to_reg,
140	};
141
142	struct mlx5e_tc_jump_state {
143	u32 jump_count;
144	bool jump_target;
145	struct mlx5_flow_attr *jumping_attr;
146
147	enum flow_action_id last_id;
148	u32 last_index;
149	};
150
151	struct mlx5e_tc_table *mlx5e_tc_table_alloc(void)
152	{
153	struct mlx5e_tc_table *tc;
154
155	tc = kvzalloc(size: sizeof(*tc), GFP_KERNEL);
156	return tc ? tc : ERR_PTR(error: -ENOMEM);
157	}
158
159	void mlx5e_tc_table_free(struct mlx5e_tc_table *tc)
160	{
161	kvfree(addr: tc);
162	}
163
164	struct mlx5_fs_chains *mlx5e_nic_chains(struct mlx5e_tc_table *tc)
165	{
166	return tc->chains;
167	}
168
169	/* To avoid false lock dependency warning set the tc_ht lock
170	* class different than the lock class of the ht being used when deleting
171	* last flow from a group and then deleting a group, we get into del_sw_flow_group()
172	* which call rhashtable_destroy on fg->ftes_hash which will take ht->mutex but
173	* it's different than the ht->mutex here.
174	*/
175	static struct lock_class_key tc_ht_lock_key;
176	static struct lock_class_key tc_ht_wq_key;
177
178	static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow);
179	static void free_flow_post_acts(struct mlx5e_tc_flow *flow);
180	static void mlx5_free_flow_attr_actions(struct mlx5e_tc_flow *flow,
181	struct mlx5_flow_attr *attr);
182
183	void
184	mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec,
185	enum mlx5e_tc_attr_to_reg type,
186	u32 val,
187	u32 mask)
188	{
189	void *headers_c = spec->match_criteria, *headers_v = spec->match_value, *fmask, *fval;
190	int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset;
191	int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset;
192	int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen;
193	u32 max_mask = GENMASK(match_len - 1, 0);
194	__be32 curr_mask_be, curr_val_be;
195	u32 curr_mask, curr_val;
196
197	fmask = headers_c + soffset;
198	fval = headers_v + soffset;
199
200	memcpy(&curr_mask_be, fmask, 4);
201	memcpy(&curr_val_be, fval, 4);
202
203	curr_mask = be32_to_cpu(curr_mask_be);
204	curr_val = be32_to_cpu(curr_val_be);
205
206	//move to correct offset
207	WARN_ON(mask > max_mask);
208	mask <<= moffset;
209	val <<= moffset;
210	max_mask <<= moffset;
211
212	//zero val and mask
213	curr_mask &= ~max_mask;
214	curr_val &= ~max_mask;
215
216	//add current to mask
217	curr_mask |= mask;
218	curr_val |= val;
219
220	//back to be32 and write
221	curr_mask_be = cpu_to_be32(curr_mask);
222	curr_val_be = cpu_to_be32(curr_val);
223
224	memcpy(fmask, &curr_mask_be, 4);
225	memcpy(fval, &curr_val_be, 4);
226
227	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
228	}
229
230	void
231	mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec,
232	enum mlx5e_tc_attr_to_reg type,
233	u32 *val,
234	u32 *mask)
235	{
236	void *headers_c = spec->match_criteria, *headers_v = spec->match_value, *fmask, *fval;
237	int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset;
238	int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset;
239	int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen;
240	u32 max_mask = GENMASK(match_len - 1, 0);
241	__be32 curr_mask_be, curr_val_be;
242	u32 curr_mask, curr_val;
243
244	fmask = headers_c + soffset;
245	fval = headers_v + soffset;
246
247	memcpy(&curr_mask_be, fmask, 4);
248	memcpy(&curr_val_be, fval, 4);
249
250	curr_mask = be32_to_cpu(curr_mask_be);
251	curr_val = be32_to_cpu(curr_val_be);
252
253	*mask = (curr_mask >> moffset) & max_mask;
254	*val = (curr_val >> moffset) & max_mask;
255	}
256
257	int
258	mlx5e_tc_match_to_reg_set_and_get_id(struct mlx5_core_dev *mdev,
259	struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts,
260	enum mlx5_flow_namespace_type ns,
261	enum mlx5e_tc_attr_to_reg type,
262	u32 data)
263	{
264	int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset;
265	int mfield = mlx5e_tc_attr_to_reg_mappings[type].mfield;
266	int mlen = mlx5e_tc_attr_to_reg_mappings[type].mlen;
267	char *modact;
268	int err;
269
270	modact = mlx5e_mod_hdr_alloc(mdev, namespace: ns, mod_hdr_acts);
271	if (IS_ERR(ptr: modact))
272	return PTR_ERR(ptr: modact);
273
274	/* Firmware has 5bit length field and 0 means 32bits */
275	if (mlen == 32)
276	mlen = 0;
277
278	MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
279	MLX5_SET(set_action_in, modact, field, mfield);
280	MLX5_SET(set_action_in, modact, offset, moffset);
281	MLX5_SET(set_action_in, modact, length, mlen);
282	MLX5_SET(set_action_in, modact, data, data);
283	err = mod_hdr_acts->num_actions;
284	mod_hdr_acts->num_actions++;
285
286	return err;
287	}
288
289	static struct mlx5e_tc_act_stats_handle *
290	get_act_stats_handle(struct mlx5e_priv *priv)
291	{
292	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
293	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
294	struct mlx5_rep_uplink_priv *uplink_priv;
295	struct mlx5e_rep_priv *uplink_rpriv;
296
297	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
298	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
299	uplink_priv = &uplink_rpriv->uplink_priv;
300
301	return uplink_priv->action_stats_handle;
302	}
303
304	return tc->action_stats_handle;
305	}
306
307	struct mlx5e_tc_int_port_priv *
308	mlx5e_get_int_port_priv(struct mlx5e_priv *priv)
309	{
310	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
311	struct mlx5_rep_uplink_priv *uplink_priv;
312	struct mlx5e_rep_priv *uplink_rpriv;
313
314	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
315	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
316	uplink_priv = &uplink_rpriv->uplink_priv;
317
318	return uplink_priv->int_port_priv;
319	}
320
321	return NULL;
322	}
323
324	struct mlx5e_flow_meters *
325	mlx5e_get_flow_meters(struct mlx5_core_dev *dev)
326	{
327	struct mlx5_eswitch *esw = dev->priv.eswitch;
328	struct mlx5_rep_uplink_priv *uplink_priv;
329	struct mlx5e_rep_priv *uplink_rpriv;
330	struct mlx5e_priv *priv;
331
332	if (is_mdev_switchdev_mode(dev)) {
333	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
334	uplink_priv = &uplink_rpriv->uplink_priv;
335	priv = netdev_priv(dev: uplink_rpriv->netdev);
336	if (!uplink_priv->flow_meters)
337	uplink_priv->flow_meters =
338	mlx5e_flow_meters_init(priv,
339	ns_type: MLX5_FLOW_NAMESPACE_FDB,
340	post_action: uplink_priv->post_act);
341	if (!IS_ERR(ptr: uplink_priv->flow_meters))
342	return uplink_priv->flow_meters;
343	}
344
345	return NULL;
346	}
347
348	static struct mlx5_tc_ct_priv *
349	get_ct_priv(struct mlx5e_priv *priv)
350	{
351	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
352	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
353	struct mlx5_rep_uplink_priv *uplink_priv;
354	struct mlx5e_rep_priv *uplink_rpriv;
355
356	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
357	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
358	uplink_priv = &uplink_rpriv->uplink_priv;
359
360	return uplink_priv->ct_priv;
361	}
362
363	return tc->ct;
364	}
365
366	static struct mlx5e_tc_psample *
367	get_sample_priv(struct mlx5e_priv *priv)
368	{
369	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
370	struct mlx5_rep_uplink_priv *uplink_priv;
371	struct mlx5e_rep_priv *uplink_rpriv;
372
373	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
374	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
375	uplink_priv = &uplink_rpriv->uplink_priv;
376
377	return uplink_priv->tc_psample;
378	}
379
380	return NULL;
381	}
382
383	static struct mlx5e_post_act *
384	get_post_action(struct mlx5e_priv *priv)
385	{
386	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
387	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
388	struct mlx5_rep_uplink_priv *uplink_priv;
389	struct mlx5e_rep_priv *uplink_rpriv;
390
391	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
392	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
393	uplink_priv = &uplink_rpriv->uplink_priv;
394
395	return uplink_priv->post_act;
396	}
397
398	return tc->post_act;
399	}
400
401	struct mlx5_flow_handle *
402	mlx5_tc_rule_insert(struct mlx5e_priv *priv,
403	struct mlx5_flow_spec *spec,
404	struct mlx5_flow_attr *attr)
405	{
406	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
407
408	if (is_mdev_switchdev_mode(dev: priv->mdev))
409	return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
410
411	return mlx5e_add_offloaded_nic_rule(priv, spec, attr);
412	}
413
414	void
415	mlx5_tc_rule_delete(struct mlx5e_priv *priv,
416	struct mlx5_flow_handle *rule,
417	struct mlx5_flow_attr *attr)
418	{
419	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
420
421	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
422	mlx5_eswitch_del_offloaded_rule(esw, rule, attr);
423	return;
424	}
425
426	mlx5e_del_offloaded_nic_rule(priv, rule, attr);
427	}
428
429	static bool
430	is_flow_meter_action(struct mlx5_flow_attr *attr)
431	{
432	return (((attr->action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) &&
433	(attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)) ||
434	attr->flags & MLX5_ATTR_FLAG_MTU);
435	}
436
437	static int
438	mlx5e_tc_add_flow_meter(struct mlx5e_priv *priv,
439	struct mlx5_flow_attr *attr)
440	{
441	struct mlx5e_post_act *post_act = get_post_action(priv);
442	struct mlx5e_post_meter_priv *post_meter;
443	enum mlx5_flow_namespace_type ns_type;
444	struct mlx5e_flow_meter_handle *meter;
445	enum mlx5e_post_meter_type type;
446
447	if (IS_ERR(ptr: post_act))
448	return PTR_ERR(ptr: post_act);
449
450	meter = mlx5e_tc_meter_replace(mdev: priv->mdev, params: &attr->meter_attr.params);
451	if (IS_ERR(ptr: meter)) {
452	mlx5_core_err(priv->mdev, "Failed to get flow meter\n");
453	return PTR_ERR(ptr: meter);
454	}
455
456	ns_type = mlx5e_tc_meter_get_namespace(flow_meters: meter->flow_meters);
457	type = meter->params.mtu ? MLX5E_POST_METER_MTU : MLX5E_POST_METER_RATE;
458	post_meter = mlx5e_post_meter_init(priv, ns_type, post_act,
459	type,
460	act_counter: meter->act_counter, drop_counter: meter->drop_counter,
461	branch_true: attr->branch_true, branch_false: attr->branch_false);
462	if (IS_ERR(ptr: post_meter)) {
463	mlx5_core_err(priv->mdev, "Failed to init post meter\n");
464	goto err_meter_init;
465	}
466
467	attr->meter_attr.meter = meter;
468	attr->meter_attr.post_meter = post_meter;
469	attr->dest_ft = mlx5e_post_meter_get_ft(post_meter);
470	attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
471
472	return 0;
473
474	err_meter_init:
475	mlx5e_tc_meter_put(meter);
476	return PTR_ERR(ptr: post_meter);
477	}
478
479	static void
480	mlx5e_tc_del_flow_meter(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
481	{
482	mlx5e_post_meter_cleanup(esw, post_meter: attr->meter_attr.post_meter);
483	mlx5e_tc_meter_put(meter: attr->meter_attr.meter);
484	}
485
486	struct mlx5_flow_handle *
487	mlx5e_tc_rule_offload(struct mlx5e_priv *priv,
488	struct mlx5_flow_spec *spec,
489	struct mlx5_flow_attr *attr)
490	{
491	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
492	int err;
493
494	if (!is_mdev_switchdev_mode(dev: priv->mdev))
495	return mlx5e_add_offloaded_nic_rule(priv, spec, attr);
496
497	if (attr->flags & MLX5_ATTR_FLAG_SAMPLE)
498	return mlx5e_tc_sample_offload(sample_priv: get_sample_priv(priv), spec, attr);
499
500	if (is_flow_meter_action(attr)) {
501	err = mlx5e_tc_add_flow_meter(priv, attr);
502	if (err)
503	return ERR_PTR(error: err);
504	}
505
506	return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
507	}
508
509	void
510	mlx5e_tc_rule_unoffload(struct mlx5e_priv *priv,
511	struct mlx5_flow_handle *rule,
512	struct mlx5_flow_attr *attr)
513	{
514	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
515
516	if (!is_mdev_switchdev_mode(dev: priv->mdev)) {
517	mlx5e_del_offloaded_nic_rule(priv, rule, attr);
518	return;
519	}
520
521	if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) {
522	mlx5e_tc_sample_unoffload(sample_priv: get_sample_priv(priv), rule, attr);
523	return;
524	}
525
526	mlx5_eswitch_del_offloaded_rule(esw, rule, attr);
527
528	if (attr->meter_attr.meter)
529	mlx5e_tc_del_flow_meter(esw, attr);
530	}
531
532	int
533	mlx5e_tc_match_to_reg_set(struct mlx5_core_dev *mdev,
534	struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts,
535	enum mlx5_flow_namespace_type ns,
536	enum mlx5e_tc_attr_to_reg type,
537	u32 data)
538	{
539	int ret = mlx5e_tc_match_to_reg_set_and_get_id(mdev, mod_hdr_acts, ns, type, data);
540
541	return ret < 0 ? ret : 0;
542	}
543
544	void mlx5e_tc_match_to_reg_mod_hdr_change(struct mlx5_core_dev *mdev,
545	struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts,
546	enum mlx5e_tc_attr_to_reg type,
547	int act_id, u32 data)
548	{
549	int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset;
550	int mfield = mlx5e_tc_attr_to_reg_mappings[type].mfield;
551	int mlen = mlx5e_tc_attr_to_reg_mappings[type].mlen;
552	char *modact;
553
554	modact = mlx5e_mod_hdr_get_item(mod_hdr_acts, pos: act_id);
555
556	/* Firmware has 5bit length field and 0 means 32bits */
557	if (mlen == 32)
558	mlen = 0;
559
560	MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
561	MLX5_SET(set_action_in, modact, field, mfield);
562	MLX5_SET(set_action_in, modact, offset, moffset);
563	MLX5_SET(set_action_in, modact, length, mlen);
564	MLX5_SET(set_action_in, modact, data, data);
565	}
566
567	struct mlx5e_hairpin {
568	struct mlx5_hairpin *pair;
569
570	struct mlx5_core_dev *func_mdev;
571	struct mlx5e_priv *func_priv;
572	u32 tdn;
573	struct mlx5e_tir direct_tir;
574
575	int num_channels;
576	u8 log_num_packets;
577	struct mlx5e_rqt indir_rqt;
578	struct mlx5e_tir indir_tir[MLX5E_NUM_INDIR_TIRS];
579	struct mlx5_ttc_table *ttc;
580	};
581
582	struct mlx5e_hairpin_entry {
583	/* a node of a hash table which keeps all the hairpin entries */
584	struct hlist_node hairpin_hlist;
585
586	/* protects flows list */
587	spinlock_t flows_lock;
588	/* flows sharing the same hairpin */
589	struct list_head flows;
590	/* hpe's that were not fully initialized when dead peer update event
591	* function traversed them.
592	*/
593	struct list_head dead_peer_wait_list;
594
595	u16 peer_vhca_id;
596	u8 prio;
597	struct mlx5e_hairpin *hp;
598	refcount_t refcnt;
599	struct completion res_ready;
600	};
601
602	static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
603	struct mlx5e_tc_flow *flow);
604
605	struct mlx5e_tc_flow *mlx5e_flow_get(struct mlx5e_tc_flow *flow)
606	{
607	if (!flow || !refcount_inc_not_zero(r: &flow->refcnt))
608	return ERR_PTR(error: -EINVAL);
609	return flow;
610	}
611
612	void mlx5e_flow_put(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow)
613	{
614	if (refcount_dec_and_test(r: &flow->refcnt)) {
615	mlx5e_tc_del_flow(priv, flow);
616	kfree_rcu(flow, rcu_head);
617	}
618	}
619
620	bool mlx5e_is_eswitch_flow(struct mlx5e_tc_flow *flow)
621	{
622	return flow_flag_test(flow, ESWITCH);
623	}
624
625	bool mlx5e_is_ft_flow(struct mlx5e_tc_flow *flow)
626	{
627	return flow_flag_test(flow, FT);
628	}
629
630	bool mlx5e_is_offloaded_flow(struct mlx5e_tc_flow *flow)
631	{
632	return flow_flag_test(flow, OFFLOADED);
633	}
634
635	int mlx5e_get_flow_namespace(struct mlx5e_tc_flow *flow)
636	{
637	return mlx5e_is_eswitch_flow(flow) ?
638	MLX5_FLOW_NAMESPACE_FDB : MLX5_FLOW_NAMESPACE_KERNEL;
639	}
640
641	static struct mlx5_core_dev *
642	get_flow_counter_dev(struct mlx5e_tc_flow *flow)
643	{
644	return mlx5e_is_eswitch_flow(flow) ? flow->attr->esw_attr->counter_dev : flow->priv->mdev;
645	}
646
647	static struct mod_hdr_tbl *
648	get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow)
649	{
650	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
651	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
652
653	return mlx5e_get_flow_namespace(flow) == MLX5_FLOW_NAMESPACE_FDB ?
654	&esw->offloads.mod_hdr :
655	&tc->mod_hdr;
656	}
657
658	int mlx5e_tc_attach_mod_hdr(struct mlx5e_priv *priv,
659	struct mlx5e_tc_flow *flow,
660	struct mlx5_flow_attr *attr)
661	{
662	struct mlx5e_mod_hdr_handle *mh;
663
664	mh = mlx5e_mod_hdr_attach(mdev: priv->mdev, tbl: get_mod_hdr_table(priv, flow),
665	namespace: mlx5e_get_flow_namespace(flow),
666	mod_hdr_acts: &attr->parse_attr->mod_hdr_acts);
667	if (IS_ERR(ptr: mh))
668	return PTR_ERR(ptr: mh);
669
670	WARN_ON(attr->modify_hdr);
671	attr->modify_hdr = mlx5e_mod_hdr_get(mh);
672	attr->mh = mh;
673
674	return 0;
675	}
676
677	void mlx5e_tc_detach_mod_hdr(struct mlx5e_priv *priv,
678	struct mlx5e_tc_flow *flow,
679	struct mlx5_flow_attr *attr)
680	{
681	/* flow wasn't fully initialized */
682	if (!attr->mh)
683	return;
684
685	mlx5e_mod_hdr_detach(mdev: priv->mdev, tbl: get_mod_hdr_table(priv, flow),
686	mh: attr->mh);
687	attr->mh = NULL;
688	}
689
690	static
691	struct mlx5_core_dev *mlx5e_hairpin_get_mdev(struct net *net, int ifindex)
692	{
693	struct mlx5_core_dev *mdev;
694	struct net_device *netdev;
695	struct mlx5e_priv *priv;
696
697	netdev = dev_get_by_index(net, ifindex);
698	if (!netdev)
699	return ERR_PTR(error: -ENODEV);
700
701	priv = netdev_priv(dev: netdev);
702	mdev = priv->mdev;
703	dev_put(dev: netdev);
704
705	/* Mirred tc action holds a refcount on the ifindex net_device (see
706	* net/sched/act_mirred.c:tcf_mirred_get_dev). So, it's okay to continue using mdev
707	* after dev_put(netdev), while we're in the context of adding a tc flow.
708	*
709	* The mdev pointer corresponds to the peer/out net_device of a hairpin. It is then
710	* stored in a hairpin object, which exists until all flows, that refer to it, get
711	* removed.
712	*
713	* On the other hand, after a hairpin object has been created, the peer net_device may
714	* be removed/unbound while there are still some hairpin flows that are using it. This
715	* case is handled by mlx5e_tc_hairpin_update_dead_peer, which is hooked to
716	* NETDEV_UNREGISTER event of the peer net_device.
717	*/
718	return mdev;
719	}
720
721	static int mlx5e_hairpin_create_transport(struct mlx5e_hairpin *hp)
722	{
723	struct mlx5e_tir_builder *builder;
724	int err;
725
726	builder = mlx5e_tir_builder_alloc(modify: false);
727	if (!builder)
728	return -ENOMEM;
729
730	err = mlx5_core_alloc_transport_domain(dev: hp->func_mdev, tdn: &hp->tdn);
731	if (err)
732	goto out;
733
734	mlx5e_tir_builder_build_inline(builder, tdn: hp->tdn, rqn: hp->pair->rqn[0]);
735	err = mlx5e_tir_init(tir: &hp->direct_tir, builder, mdev: hp->func_mdev, reg: false);
736	if (err)
737	goto create_tir_err;
738
739	out:
740	mlx5e_tir_builder_free(builder);
741	return err;
742
743	create_tir_err:
744	mlx5_core_dealloc_transport_domain(dev: hp->func_mdev, tdn: hp->tdn);
745
746	goto out;
747	}
748
749	static void mlx5e_hairpin_destroy_transport(struct mlx5e_hairpin *hp)
750	{
751	mlx5e_tir_destroy(tir: &hp->direct_tir);
752	mlx5_core_dealloc_transport_domain(dev: hp->func_mdev, tdn: hp->tdn);
753	}
754
755	static int mlx5e_hairpin_create_indirect_rqt(struct mlx5e_hairpin *hp)
756	{
757	struct mlx5e_priv *priv = hp->func_priv;
758	struct mlx5_core_dev *mdev = priv->mdev;
759	struct mlx5e_rss_params_indir indir;
760	int err;
761
762	err = mlx5e_rss_params_indir_init(indir: &indir, mdev,
763	actual_table_size: mlx5e_rqt_size(mdev, num_channels: hp->num_channels),
764	max_table_size: mlx5e_rqt_size(mdev, num_channels: hp->num_channels));
765	if (err)
766	return err;
767
768	mlx5e_rss_params_indir_init_uniform(indir: &indir, num_channels: hp->num_channels);
769	err = mlx5e_rqt_init_indir(rqt: &hp->indir_rqt, mdev, rqns: hp->pair->rqn, NULL, num_rqns: hp->num_channels,
770	hfunc: mlx5e_rx_res_get_current_hash(res: priv->rx_res).hfunc,
771	indir: &indir);
772
773	mlx5e_rss_params_indir_cleanup(indir: &indir);
774	return err;
775	}
776
777	static int mlx5e_hairpin_create_indirect_tirs(struct mlx5e_hairpin *hp)
778	{
779	struct mlx5e_priv *priv = hp->func_priv;
780	struct mlx5e_rss_params_hash rss_hash;
781	enum mlx5_traffic_types tt, max_tt;
782	struct mlx5e_tir_builder *builder;
783	int err = 0;
784
785	builder = mlx5e_tir_builder_alloc(modify: false);
786	if (!builder)
787	return -ENOMEM;
788
789	rss_hash = mlx5e_rx_res_get_current_hash(res: priv->rx_res);
790
791	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
792	struct mlx5e_rss_params_traffic_type rss_tt;
793
794	rss_tt = mlx5e_rss_get_default_tt_config(tt);
795
796	mlx5e_tir_builder_build_rqt(builder, tdn: hp->tdn,
797	rqtn: mlx5e_rqt_get_rqtn(rqt: &hp->indir_rqt),
798	inner_ft_support: false);
799	mlx5e_tir_builder_build_rss(builder, rss_hash: &rss_hash, rss_tt: &rss_tt, inner: false);
800
801	err = mlx5e_tir_init(tir: &hp->indir_tir[tt], builder, mdev: hp->func_mdev, reg: false);
802	if (err) {
803	mlx5_core_warn(hp->func_mdev, "create indirect tirs failed, %d\n", err);
804	goto err_destroy_tirs;
805	}
806
807	mlx5e_tir_builder_clear(builder);
808	}
809
810	out:
811	mlx5e_tir_builder_free(builder);
812	return err;
813
814	err_destroy_tirs:
815	max_tt = tt;
816	for (tt = 0; tt < max_tt; tt++)
817	mlx5e_tir_destroy(tir: &hp->indir_tir[tt]);
818
819	goto out;
820	}
821
822	static void mlx5e_hairpin_destroy_indirect_tirs(struct mlx5e_hairpin *hp)
823	{
824	int tt;
825
826	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
827	mlx5e_tir_destroy(tir: &hp->indir_tir[tt]);
828	}
829
830	static void mlx5e_hairpin_set_ttc_params(struct mlx5e_hairpin *hp,
831	struct ttc_params *ttc_params)
832	{
833	struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
834	int tt;
835
836	memset(ttc_params, 0, sizeof(*ttc_params));
837
838	ttc_params->ns = mlx5_get_flow_namespace(dev: hp->func_mdev,
839	type: MLX5_FLOW_NAMESPACE_KERNEL);
840	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
841	ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
842	ttc_params->dests[tt].tir_num =
843	tt == MLX5_TT_ANY ?
844	mlx5e_tir_get_tirn(tir: &hp->direct_tir) :
845	mlx5e_tir_get_tirn(tir: &hp->indir_tir[tt]);
846	}
847
848	ft_attr->level = MLX5E_TC_TTC_FT_LEVEL;
849	ft_attr->prio = MLX5E_TC_PRIO;
850	}
851
852	static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp)
853	{
854	struct mlx5e_priv *priv = hp->func_priv;
855	struct ttc_params ttc_params;
856	struct mlx5_ttc_table *ttc;
857	int err;
858
859	err = mlx5e_hairpin_create_indirect_rqt(hp);
860	if (err)
861	return err;
862
863	err = mlx5e_hairpin_create_indirect_tirs(hp);
864	if (err)
865	goto err_create_indirect_tirs;
866
867	mlx5e_hairpin_set_ttc_params(hp, ttc_params: &ttc_params);
868	hp->ttc = mlx5_create_ttc_table(dev: priv->mdev, params: &ttc_params);
869	if (IS_ERR(ptr: hp->ttc)) {
870	err = PTR_ERR(ptr: hp->ttc);
871	goto err_create_ttc_table;
872	}
873
874	ttc = mlx5e_fs_get_ttc(fs: priv->fs, inner: false);
875	netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n",
876	hp->num_channels,
877	mlx5_get_ttc_flow_table(ttc)->id);
878
879	return 0;
880
881	err_create_ttc_table:
882	mlx5e_hairpin_destroy_indirect_tirs(hp);
883	err_create_indirect_tirs:
884	mlx5e_rqt_destroy(rqt: &hp->indir_rqt);
885
886	return err;
887	}
888
889	static void mlx5e_hairpin_rss_cleanup(struct mlx5e_hairpin *hp)
890	{
891	mlx5_destroy_ttc_table(ttc: hp->ttc);
892	mlx5e_hairpin_destroy_indirect_tirs(hp);
893	mlx5e_rqt_destroy(rqt: &hp->indir_rqt);
894	}
895
896	static struct mlx5e_hairpin *
897	mlx5e_hairpin_create(struct mlx5e_priv *priv, struct mlx5_hairpin_params *params,
898	int peer_ifindex)
899	{
900	struct mlx5_core_dev *func_mdev, *peer_mdev;
901	struct mlx5e_hairpin *hp;
902	struct mlx5_hairpin *pair;
903	int err;
904
905	hp = kzalloc(size: sizeof(*hp), GFP_KERNEL);
906	if (!hp)
907	return ERR_PTR(error: -ENOMEM);
908
909	func_mdev = priv->mdev;
910	peer_mdev = mlx5e_hairpin_get_mdev(net: dev_net(dev: priv->netdev), ifindex: peer_ifindex);
911	if (IS_ERR(ptr: peer_mdev)) {
912	err = PTR_ERR(ptr: peer_mdev);
913	goto create_pair_err;
914	}
915
916	pair = mlx5_core_hairpin_create(func_mdev, peer_mdev, params);
917	if (IS_ERR(ptr: pair)) {
918	err = PTR_ERR(ptr: pair);
919	goto create_pair_err;
920	}
921	hp->pair = pair;
922	hp->func_mdev = func_mdev;
923	hp->func_priv = priv;
924	hp->num_channels = params->num_channels;
925	hp->log_num_packets = params->log_num_packets;
926
927	err = mlx5e_hairpin_create_transport(hp);
928	if (err)
929	goto create_transport_err;
930
931	if (hp->num_channels > 1) {
932	err = mlx5e_hairpin_rss_init(hp);
933	if (err)
934	goto rss_init_err;
935	}
936
937	return hp;
938
939	rss_init_err:
940	mlx5e_hairpin_destroy_transport(hp);
941	create_transport_err:
942	mlx5_core_hairpin_destroy(pair: hp->pair);
943	create_pair_err:
944	kfree(objp: hp);
945	return ERR_PTR(error: err);
946	}
947
948	static void mlx5e_hairpin_destroy(struct mlx5e_hairpin *hp)
949	{
950	if (hp->num_channels > 1)
951	mlx5e_hairpin_rss_cleanup(hp);
952	mlx5e_hairpin_destroy_transport(hp);
953	mlx5_core_hairpin_destroy(pair: hp->pair);
954	kvfree(addr: hp);
955	}
956
957	static inline u32 hash_hairpin_info(u16 peer_vhca_id, u8 prio)
958	{
959	return (peer_vhca_id << 16 | prio);
960	}
961
962	static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv,
963	u16 peer_vhca_id, u8 prio)
964	{
965	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
966	struct mlx5e_hairpin_entry *hpe;
967	u32 hash_key = hash_hairpin_info(peer_vhca_id, prio);
968
969	hash_for_each_possible(tc->hairpin_tbl, hpe,
970	hairpin_hlist, hash_key) {
971	if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio) {
972	refcount_inc(r: &hpe->refcnt);
973	return hpe;
974	}
975	}
976
977	return NULL;
978	}
979
980	static void mlx5e_hairpin_put(struct mlx5e_priv *priv,
981	struct mlx5e_hairpin_entry *hpe)
982	{
983	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
984	/* no more hairpin flows for us, release the hairpin pair */
985	if (!refcount_dec_and_mutex_lock(r: &hpe->refcnt, lock: &tc->hairpin_tbl_lock))
986	return;
987	hash_del(node: &hpe->hairpin_hlist);
988	mutex_unlock(lock: &tc->hairpin_tbl_lock);
989
990	if (!IS_ERR_OR_NULL(ptr: hpe->hp)) {
991	netdev_dbg(priv->netdev, "del hairpin: peer %s\n",
992	dev_name(hpe->hp->pair->peer_mdev->device));
993
994	mlx5e_hairpin_destroy(hp: hpe->hp);
995	}
996
997	WARN_ON(!list_empty(&hpe->flows));
998	kfree(objp: hpe);
999	}
1000
1001	#define UNKNOWN_MATCH_PRIO 8
1002
1003	static int mlx5e_hairpin_get_prio(struct mlx5e_priv *priv,
1004	struct mlx5_flow_spec *spec, u8 *match_prio,
1005	struct netlink_ext_ack *extack)
1006	{
1007	void *headers_c, *headers_v;
1008	u8 prio_val, prio_mask = 0;
1009	bool vlan_present;
1010
1011	#ifdef CONFIG_MLX5_CORE_EN_DCB
1012	if (priv->dcbx_dp.trust_state != MLX5_QPTS_TRUST_PCP) {
1013	NL_SET_ERR_MSG_MOD(extack,
1014	"only PCP trust state supported for hairpin");
1015	return -EOPNOTSUPP;
1016	}
1017	#endif
1018	headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers);
1019	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
1020
1021	vlan_present = MLX5_GET(fte_match_set_lyr_2_4, headers_v, cvlan_tag);
1022	if (vlan_present) {
1023	prio_mask = MLX5_GET(fte_match_set_lyr_2_4, headers_c, first_prio);
1024	prio_val = MLX5_GET(fte_match_set_lyr_2_4, headers_v, first_prio);
1025	}
1026
1027	if (!vlan_present || !prio_mask) {
1028	prio_val = UNKNOWN_MATCH_PRIO;
1029	} else if (prio_mask != 0x7) {
1030	NL_SET_ERR_MSG_MOD(extack,
1031	"masked priority match not supported for hairpin");
1032	return -EOPNOTSUPP;
1033	}
1034
1035	*match_prio = prio_val;
1036	return 0;
1037	}
1038
1039	static int debugfs_hairpin_num_active_get(void *data, u64 *val)
1040	{
1041	struct mlx5e_tc_table *tc = data;
1042	struct mlx5e_hairpin_entry *hpe;
1043	u32 cnt = 0;
1044	u32 bkt;
1045
1046	mutex_lock(&tc->hairpin_tbl_lock);
1047	hash_for_each(tc->hairpin_tbl, bkt, hpe, hairpin_hlist)
1048	cnt++;
1049	mutex_unlock(lock: &tc->hairpin_tbl_lock);
1050
1051	*val = cnt;
1052
1053	return 0;
1054	}
1055	DEFINE_DEBUGFS_ATTRIBUTE(fops_hairpin_num_active,
1056	debugfs_hairpin_num_active_get, NULL, "%llu\n");
1057
1058	static int debugfs_hairpin_table_dump_show(struct seq_file *file, void *priv)
1059
1060	{
1061	struct mlx5e_tc_table *tc = file->private;
1062	struct mlx5e_hairpin_entry *hpe;
1063	u32 bkt;
1064
1065	mutex_lock(&tc->hairpin_tbl_lock);
1066	hash_for_each(tc->hairpin_tbl, bkt, hpe, hairpin_hlist)
1067	seq_printf(m: file,
1068	fmt: "Hairpin peer_vhca_id %u prio %u refcnt %u num_channels %u num_packets %lu\n",
1069	hpe->peer_vhca_id, hpe->prio,
1070	refcount_read(r: &hpe->refcnt), hpe->hp->num_channels,
1071	BIT(hpe->hp->log_num_packets));
1072	mutex_unlock(lock: &tc->hairpin_tbl_lock);
1073
1074	return 0;
1075	}
1076	DEFINE_SHOW_ATTRIBUTE(debugfs_hairpin_table_dump);
1077
1078	static void mlx5e_tc_debugfs_init(struct mlx5e_tc_table *tc,
1079	struct dentry *dfs_root)
1080	{
1081	if (IS_ERR_OR_NULL(ptr: dfs_root))
1082	return;
1083
1084	tc->dfs_root = debugfs_create_dir(name: "tc", parent: dfs_root);
1085
1086	debugfs_create_file(name: "hairpin_num_active", mode: 0444, parent: tc->dfs_root, data: tc,
1087	fops: &fops_hairpin_num_active);
1088	debugfs_create_file(name: "hairpin_table_dump", mode: 0444, parent: tc->dfs_root, data: tc,
1089	fops: &debugfs_hairpin_table_dump_fops);
1090	}
1091
1092	static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
1093	struct mlx5e_tc_flow *flow,
1094	struct mlx5e_tc_flow_parse_attr *parse_attr,
1095	struct netlink_ext_ack *extack)
1096	{
1097	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
1098	struct devlink *devlink = priv_to_devlink(priv: priv->mdev);
1099	int peer_ifindex = parse_attr->mirred_ifindex[0];
1100	union devlink_param_value val = {};
1101	struct mlx5_hairpin_params params;
1102	struct mlx5_core_dev *peer_mdev;
1103	struct mlx5e_hairpin_entry *hpe;
1104	struct mlx5e_hairpin *hp;
1105	u8 match_prio;
1106	u16 peer_id;
1107	int err;
1108
1109	peer_mdev = mlx5e_hairpin_get_mdev(net: dev_net(dev: priv->netdev), ifindex: peer_ifindex);
1110	if (IS_ERR(ptr: peer_mdev)) {
1111	NL_SET_ERR_MSG_MOD(extack, "invalid ifindex of mirred device");
1112	return PTR_ERR(ptr: peer_mdev);
1113	}
1114
1115	if (!MLX5_CAP_GEN(priv->mdev, hairpin) || !MLX5_CAP_GEN(peer_mdev, hairpin)) {
1116	NL_SET_ERR_MSG_MOD(extack, "hairpin is not supported");
1117	return -EOPNOTSUPP;
1118	}
1119
1120	peer_id = MLX5_CAP_GEN(peer_mdev, vhca_id);
1121	err = mlx5e_hairpin_get_prio(priv, spec: &parse_attr->spec, match_prio: &match_prio,
1122	extack);
1123	if (err)
1124	return err;
1125
1126	mutex_lock(&tc->hairpin_tbl_lock);
1127	hpe = mlx5e_hairpin_get(priv, peer_vhca_id: peer_id, prio: match_prio);
1128	if (hpe) {
1129	mutex_unlock(lock: &tc->hairpin_tbl_lock);
1130	wait_for_completion(&hpe->res_ready);
1131
1132	if (IS_ERR(ptr: hpe->hp)) {
1133	err = -EREMOTEIO;
1134	goto out_err;
1135	}
1136	goto attach_flow;
1137	}
1138
1139	hpe = kzalloc(size: sizeof(*hpe), GFP_KERNEL);
1140	if (!hpe) {
1141	mutex_unlock(lock: &tc->hairpin_tbl_lock);
1142	return -ENOMEM;
1143	}
1144
1145	spin_lock_init(&hpe->flows_lock);
1146	INIT_LIST_HEAD(list: &hpe->flows);
1147	INIT_LIST_HEAD(list: &hpe->dead_peer_wait_list);
1148	hpe->peer_vhca_id = peer_id;
1149	hpe->prio = match_prio;
1150	refcount_set(r: &hpe->refcnt, n: 1);
1151	init_completion(x: &hpe->res_ready);
1152
1153	hash_add(tc->hairpin_tbl, &hpe->hairpin_hlist,
1154	hash_hairpin_info(peer_id, match_prio));
1155	mutex_unlock(lock: &tc->hairpin_tbl_lock);
1156
1157	err = devl_param_driverinit_value_get(
1158	devlink, param_id: MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE, val: &val);
1159	if (err) {
1160	err = -ENOMEM;
1161	goto out_err;
1162	}
1163
1164	params.log_num_packets = ilog2(val.vu32);
1165	params.log_data_size =
1166	clamp_t(u32,
1167	params.log_num_packets +
1168	MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(priv->mdev),
1169	MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz),
1170	MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz));
1171
1172	params.q_counter = priv->q_counter[0];
1173	err = devl_param_driverinit_value_get(
1174	devlink, param_id: MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES, val: &val);
1175	if (err) {
1176	err = -ENOMEM;
1177	goto out_err;
1178	}
1179
1180	params.num_channels = val.vu32;
1181
1182	hp = mlx5e_hairpin_create(priv, params: &params, peer_ifindex);
1183	hpe->hp = hp;
1184	complete_all(&hpe->res_ready);
1185	if (IS_ERR(ptr: hp)) {
1186	err = PTR_ERR(ptr: hp);
1187	goto out_err;
1188	}
1189
1190	netdev_dbg(priv->netdev, "add hairpin: tirn %x rqn %x peer %s sqn %x prio %d (log) data %d packets %d\n",
1191	mlx5e_tir_get_tirn(&hp->direct_tir), hp->pair->rqn[0],
1192	dev_name(hp->pair->peer_mdev->device),
1193	hp->pair->sqn[0], match_prio, params.log_data_size, params.log_num_packets);
1194
1195	attach_flow:
1196	if (hpe->hp->num_channels > 1) {
1197	flow_flag_set(flow, HAIRPIN_RSS);
1198	flow->attr->nic_attr->hairpin_ft =
1199	mlx5_get_ttc_flow_table(ttc: hpe->hp->ttc);
1200	} else {
1201	flow->attr->nic_attr->hairpin_tirn = mlx5e_tir_get_tirn(tir: &hpe->hp->direct_tir);
1202	}
1203
1204	flow->hpe = hpe;
1205	spin_lock(lock: &hpe->flows_lock);
1206	list_add(new: &flow->hairpin, head: &hpe->flows);
1207	spin_unlock(lock: &hpe->flows_lock);
1208
1209	return 0;
1210
1211	out_err:
1212	mlx5e_hairpin_put(priv, hpe);
1213	return err;
1214	}
1215
1216	static void mlx5e_hairpin_flow_del(struct mlx5e_priv *priv,
1217	struct mlx5e_tc_flow *flow)
1218	{
1219	/* flow wasn't fully initialized */
1220	if (!flow->hpe)
1221	return;
1222
1223	spin_lock(lock: &flow->hpe->flows_lock);
1224	list_del(entry: &flow->hairpin);
1225	spin_unlock(lock: &flow->hpe->flows_lock);
1226
1227	mlx5e_hairpin_put(priv, hpe: flow->hpe);
1228	flow->hpe = NULL;
1229	}
1230
1231	struct mlx5_flow_handle *
1232	mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv,
1233	struct mlx5_flow_spec *spec,
1234	struct mlx5_flow_attr *attr)
1235	{
1236	struct mlx5_flow_context *flow_context = &spec->flow_context;
1237	struct mlx5e_vlan_table *vlan = mlx5e_fs_get_vlan(fs: priv->fs);
1238	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
1239	struct mlx5_nic_flow_attr *nic_attr = attr->nic_attr;
1240	struct mlx5_flow_destination dest[2] = {};
1241	struct mlx5_fs_chains *nic_chains;
1242	struct mlx5_flow_act flow_act = {
1243	.action = attr->action,
1244	.flags = FLOW_ACT_NO_APPEND,
1245	};
1246	struct mlx5_flow_handle *rule;
1247	struct mlx5_flow_table *ft;
1248	int dest_ix = 0;
1249
1250	nic_chains = mlx5e_nic_chains(tc);
1251	flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1252	flow_context->flow_tag = nic_attr->flow_tag;
1253
1254	if (attr->dest_ft) {
1255	dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1256	dest[dest_ix].ft = attr->dest_ft;
1257	dest_ix++;
1258	} else if (nic_attr->hairpin_ft) {
1259	dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1260	dest[dest_ix].ft = nic_attr->hairpin_ft;
1261	dest_ix++;
1262	} else if (nic_attr->hairpin_tirn) {
1263	dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
1264	dest[dest_ix].tir_num = nic_attr->hairpin_tirn;
1265	dest_ix++;
1266	} else if (attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
1267	dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1268	if (attr->dest_chain) {
1269	dest[dest_ix].ft = mlx5_chains_get_table(chains: nic_chains,
1270	chain: attr->dest_chain, prio: 1,
1271	level: MLX5E_TC_FT_LEVEL);
1272	if (IS_ERR(ptr: dest[dest_ix].ft))
1273	return ERR_CAST(ptr: dest[dest_ix].ft);
1274	} else {
1275	dest[dest_ix].ft = mlx5e_vlan_get_flowtable(vlan);
1276	}
1277	dest_ix++;
1278	}
1279
1280	if (dest[0].type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE &&
1281	MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ignore_flow_level))
1282	flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
1283
1284	if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1285	dest[dest_ix].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
1286	dest[dest_ix].counter_id = mlx5_fc_id(counter: attr->counter);
1287	dest_ix++;
1288	}
1289
1290	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
1291	flow_act.modify_hdr = attr->modify_hdr;
1292
1293	mutex_lock(&tc->t_lock);
1294	if (IS_ERR_OR_NULL(ptr: tc->t)) {
1295	/* Create the root table here if doesn't exist yet */
1296	tc->t =
1297	mlx5_chains_get_table(chains: nic_chains, chain: 0, prio: 1, level: MLX5E_TC_FT_LEVEL);
1298
1299	if (IS_ERR(ptr: tc->t)) {
1300	mutex_unlock(lock: &tc->t_lock);
1301	netdev_err(dev: priv->netdev,
1302	format: "Failed to create tc offload table\n");
1303	rule = ERR_CAST(ptr: tc->t);
1304	goto err_ft_get;
1305	}
1306	}
1307	mutex_unlock(lock: &tc->t_lock);
1308
1309	if (attr->chain || attr->prio)
1310	ft = mlx5_chains_get_table(chains: nic_chains,
1311	chain: attr->chain, prio: attr->prio,
1312	level: MLX5E_TC_FT_LEVEL);
1313	else
1314	ft = attr->ft;
1315
1316	if (IS_ERR(ptr: ft)) {
1317	rule = ERR_CAST(ptr: ft);
1318	goto err_ft_get;
1319	}
1320
1321	if (attr->outer_match_level != MLX5_MATCH_NONE)
1322	spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
1323
1324	rule = mlx5_add_flow_rules(ft, spec,
1325	flow_act: &flow_act, dest, num_dest: dest_ix);
1326	if (IS_ERR(ptr: rule))
1327	goto err_rule;
1328
1329	return rule;
1330
1331	err_rule:
1332	if (attr->chain || attr->prio)
1333	mlx5_chains_put_table(chains: nic_chains,
1334	chain: attr->chain, prio: attr->prio,
1335	level: MLX5E_TC_FT_LEVEL);
1336	err_ft_get:
1337	if (attr->dest_chain)
1338	mlx5_chains_put_table(chains: nic_chains,
1339	chain: attr->dest_chain, prio: 1,
1340	level: MLX5E_TC_FT_LEVEL);
1341
1342	return ERR_CAST(ptr: rule);
1343	}
1344
1345	static int
1346	alloc_flow_attr_counter(struct mlx5_core_dev *counter_dev,
1347	struct mlx5_flow_attr *attr)
1348
1349	{
1350	struct mlx5_fc *counter;
1351
1352	counter = mlx5_fc_create(dev: counter_dev, aging: true);
1353	if (IS_ERR(ptr: counter))
1354	return PTR_ERR(ptr: counter);
1355
1356	attr->counter = counter;
1357	return 0;
1358	}
1359
1360	static int
1361	mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv,
1362	struct mlx5e_tc_flow *flow,
1363	struct netlink_ext_ack *extack)
1364	{
1365	struct mlx5e_tc_flow_parse_attr *parse_attr;
1366	struct mlx5_flow_attr *attr = flow->attr;
1367	struct mlx5_core_dev *dev = priv->mdev;
1368	int err;
1369
1370	parse_attr = attr->parse_attr;
1371
1372	if (flow_flag_test(flow, HAIRPIN)) {
1373	err = mlx5e_hairpin_flow_add(priv, flow, parse_attr, extack);
1374	if (err)
1375	return err;
1376	}
1377
1378	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1379	err = alloc_flow_attr_counter(counter_dev: dev, attr);
1380	if (err)
1381	return err;
1382	}
1383
1384	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
1385	err = mlx5e_tc_attach_mod_hdr(priv, flow, attr);
1386	if (err)
1387	return err;
1388	}
1389
1390	flow->rule[0] = mlx5e_add_offloaded_nic_rule(priv, spec: &parse_attr->spec, attr);
1391	return PTR_ERR_OR_ZERO(ptr: flow->rule[0]);
1392	}
1393
1394	void mlx5e_del_offloaded_nic_rule(struct mlx5e_priv *priv,
1395	struct mlx5_flow_handle *rule,
1396	struct mlx5_flow_attr *attr)
1397	{
1398	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
1399	struct mlx5_fs_chains *nic_chains;
1400
1401	nic_chains = mlx5e_nic_chains(tc);
1402	mlx5_del_flow_rules(fr: rule);
1403
1404	if (attr->chain || attr->prio)
1405	mlx5_chains_put_table(chains: nic_chains, chain: attr->chain, prio: attr->prio,
1406	level: MLX5E_TC_FT_LEVEL);
1407
1408	if (attr->dest_chain)
1409	mlx5_chains_put_table(chains: nic_chains, chain: attr->dest_chain, prio: 1,
1410	level: MLX5E_TC_FT_LEVEL);
1411	}
1412
1413	static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
1414	struct mlx5e_tc_flow *flow)
1415	{
1416	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
1417	struct mlx5_flow_attr *attr = flow->attr;
1418
1419	flow_flag_clear(flow, OFFLOADED);
1420
1421	if (!IS_ERR_OR_NULL(ptr: flow->rule[0]))
1422	mlx5e_del_offloaded_nic_rule(priv, rule: flow->rule[0], attr);
1423
1424	/* Remove root table if no rules are left to avoid
1425	* extra steering hops.
1426	*/
1427	mutex_lock(&tc->t_lock);
1428	if (!mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD)) &&
1429	!IS_ERR_OR_NULL(ptr: tc->t)) {
1430	mlx5_chains_put_table(chains: mlx5e_nic_chains(tc), chain: 0, prio: 1, level: MLX5E_TC_FT_LEVEL);
1431	tc->t = NULL;
1432	}
1433	mutex_unlock(lock: &tc->t_lock);
1434
1435	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
1436	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &attr->parse_attr->mod_hdr_acts);
1437	mlx5e_tc_detach_mod_hdr(priv, flow, attr);
1438	}
1439
1440	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
1441	mlx5_fc_destroy(dev: priv->mdev, counter: attr->counter);
1442
1443	if (flow_flag_test(flow, HAIRPIN))
1444	mlx5e_hairpin_flow_del(priv, flow);
1445
1446	free_flow_post_acts(flow);
1447	mlx5_tc_ct_delete_flow(priv: get_ct_priv(priv: flow->priv), attr);
1448
1449	kvfree(addr: attr->parse_attr);
1450	kfree(objp: flow->attr);
1451	}
1452
1453	struct mlx5_flow_handle *
1454	mlx5e_tc_offload_fdb_rules(struct mlx5_eswitch *esw,
1455	struct mlx5e_tc_flow *flow,
1456	struct mlx5_flow_spec *spec,
1457	struct mlx5_flow_attr *attr)
1458	{
1459	struct mlx5_flow_handle *rule;
1460
1461	if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)
1462	return mlx5_eswitch_add_offloaded_rule(esw, spec, attr);
1463
1464	rule = mlx5e_tc_rule_offload(priv: flow->priv, spec, attr);
1465
1466	if (IS_ERR(ptr: rule))
1467	return rule;
1468
1469	if (attr->esw_attr->split_count) {
1470	flow->rule[1] = mlx5_eswitch_add_fwd_rule(esw, spec, attr);
1471	if (IS_ERR(ptr: flow->rule[1]))
1472	goto err_rule1;
1473	}
1474
1475	return rule;
1476
1477	err_rule1:
1478	mlx5e_tc_rule_unoffload(priv: flow->priv, rule, attr);
1479	return flow->rule[1];
1480	}
1481
1482	void mlx5e_tc_unoffload_fdb_rules(struct mlx5_eswitch *esw,
1483	struct mlx5e_tc_flow *flow,
1484	struct mlx5_flow_attr *attr)
1485	{
1486	flow_flag_clear(flow, OFFLOADED);
1487
1488	if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)
1489	return mlx5_eswitch_del_offloaded_rule(esw, rule: flow->rule[0], attr);
1490
1491	if (attr->esw_attr->split_count)
1492	mlx5_eswitch_del_fwd_rule(esw, rule: flow->rule[1], attr);
1493
1494	mlx5e_tc_rule_unoffload(priv: flow->priv, rule: flow->rule[0], attr);
1495	}
1496
1497	struct mlx5_flow_handle *
1498	mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch *esw,
1499	struct mlx5e_tc_flow *flow,
1500	struct mlx5_flow_spec *spec)
1501	{
1502	struct mlx5e_tc_mod_hdr_acts mod_acts = {};
1503	struct mlx5e_mod_hdr_handle *mh = NULL;
1504	struct mlx5_flow_attr *slow_attr;
1505	struct mlx5_flow_handle *rule;
1506	bool fwd_and_modify_cap;
1507	u32 chain_mapping = 0;
1508	int err;
1509
1510	slow_attr = mlx5_alloc_flow_attr(type: MLX5_FLOW_NAMESPACE_FDB);
1511	if (!slow_attr)
1512	return ERR_PTR(error: -ENOMEM);
1513
1514	memcpy(slow_attr, flow->attr, ESW_FLOW_ATTR_SZ);
1515	slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1516	slow_attr->esw_attr->split_count = 0;
1517	slow_attr->flags |= MLX5_ATTR_FLAG_SLOW_PATH;
1518
1519	fwd_and_modify_cap = MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table);
1520	if (!fwd_and_modify_cap)
1521	goto skip_restore;
1522
1523	err = mlx5_chains_get_chain_mapping(esw_chains(esw), chain: flow->attr->chain, chain_mapping: &chain_mapping);
1524	if (err)
1525	goto err_get_chain;
1526
1527	err = mlx5e_tc_match_to_reg_set(mdev: esw->dev, mod_hdr_acts: &mod_acts, ns: MLX5_FLOW_NAMESPACE_FDB,
1528	type: MAPPED_OBJ_TO_REG, data: chain_mapping);
1529	if (err)
1530	goto err_reg_set;
1531
1532	mh = mlx5e_mod_hdr_attach(mdev: esw->dev, tbl: get_mod_hdr_table(priv: flow->priv, flow),
1533	namespace: MLX5_FLOW_NAMESPACE_FDB, mod_hdr_acts: &mod_acts);
1534	if (IS_ERR(ptr: mh)) {
1535	err = PTR_ERR(ptr: mh);
1536	goto err_attach;
1537	}
1538
1539	slow_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1540	slow_attr->modify_hdr = mlx5e_mod_hdr_get(mh);
1541
1542	skip_restore:
1543	rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, attr: slow_attr);
1544	if (IS_ERR(ptr: rule)) {
1545	err = PTR_ERR(ptr: rule);
1546	goto err_offload;
1547	}
1548
1549	flow->attr->slow_mh = mh;
1550	flow->chain_mapping = chain_mapping;
1551	flow_flag_set(flow, SLOW);
1552
1553	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &mod_acts);
1554	kfree(objp: slow_attr);
1555
1556	return rule;
1557
1558	err_offload:
1559	if (fwd_and_modify_cap)
1560	mlx5e_mod_hdr_detach(mdev: esw->dev, tbl: get_mod_hdr_table(priv: flow->priv, flow), mh);
1561	err_attach:
1562	err_reg_set:
1563	if (fwd_and_modify_cap)
1564	mlx5_chains_put_chain_mapping(esw_chains(esw), chain_mapping);
1565	err_get_chain:
1566	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &mod_acts);
1567	kfree(objp: slow_attr);
1568	return ERR_PTR(error: err);
1569	}
1570
1571	void mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw,
1572	struct mlx5e_tc_flow *flow)
1573	{
1574	struct mlx5e_mod_hdr_handle *slow_mh = flow->attr->slow_mh;
1575	struct mlx5_flow_attr *slow_attr;
1576
1577	slow_attr = mlx5_alloc_flow_attr(type: MLX5_FLOW_NAMESPACE_FDB);
1578	if (!slow_attr) {
1579	mlx5_core_warn(flow->priv->mdev, "Unable to alloc attr to unoffload slow path rule\n");
1580	return;
1581	}
1582
1583	memcpy(slow_attr, flow->attr, ESW_FLOW_ATTR_SZ);
1584	slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1585	slow_attr->esw_attr->split_count = 0;
1586	slow_attr->flags |= MLX5_ATTR_FLAG_SLOW_PATH;
1587	if (slow_mh) {
1588	slow_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1589	slow_attr->modify_hdr = mlx5e_mod_hdr_get(mh: slow_mh);
1590	}
1591	mlx5e_tc_unoffload_fdb_rules(esw, flow, attr: slow_attr);
1592	if (slow_mh) {
1593	mlx5e_mod_hdr_detach(mdev: esw->dev, tbl: get_mod_hdr_table(priv: flow->priv, flow), mh: slow_mh);
1594	mlx5_chains_put_chain_mapping(esw_chains(esw), chain_mapping: flow->chain_mapping);
1595	flow->chain_mapping = 0;
1596	flow->attr->slow_mh = NULL;
1597	}
1598	flow_flag_clear(flow, SLOW);
1599	kfree(objp: slow_attr);
1600	}
1601
1602	/* Caller must obtain uplink_priv->unready_flows_lock mutex before calling this
1603	* function.
1604	*/
1605	static void unready_flow_add(struct mlx5e_tc_flow *flow,
1606	struct list_head *unready_flows)
1607	{
1608	flow_flag_set(flow, NOT_READY);
1609	list_add_tail(new: &flow->unready, head: unready_flows);
1610	}
1611
1612	/* Caller must obtain uplink_priv->unready_flows_lock mutex before calling this
1613	* function.
1614	*/
1615	static void unready_flow_del(struct mlx5e_tc_flow *flow)
1616	{
1617	list_del(entry: &flow->unready);
1618	flow_flag_clear(flow, NOT_READY);
1619	}
1620
1621	static void add_unready_flow(struct mlx5e_tc_flow *flow)
1622	{
1623	struct mlx5_rep_uplink_priv *uplink_priv;
1624	struct mlx5e_rep_priv *rpriv;
1625	struct mlx5_eswitch *esw;
1626
1627	esw = flow->priv->mdev->priv.eswitch;
1628	rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
1629	uplink_priv = &rpriv->uplink_priv;
1630
1631	mutex_lock(&uplink_priv->unready_flows_lock);
1632	unready_flow_add(flow, unready_flows: &uplink_priv->unready_flows);
1633	mutex_unlock(lock: &uplink_priv->unready_flows_lock);
1634	}
1635
1636	static void remove_unready_flow(struct mlx5e_tc_flow *flow)
1637	{
1638	struct mlx5_rep_uplink_priv *uplink_priv;
1639	struct mlx5e_rep_priv *rpriv;
1640	struct mlx5_eswitch *esw;
1641
1642	esw = flow->priv->mdev->priv.eswitch;
1643	rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
1644	uplink_priv = &rpriv->uplink_priv;
1645
1646	mutex_lock(&uplink_priv->unready_flows_lock);
1647	if (flow_flag_test(flow, NOT_READY))
1648	unready_flow_del(flow);
1649	mutex_unlock(lock: &uplink_priv->unready_flows_lock);
1650	}
1651
1652	bool mlx5e_tc_is_vf_tunnel(struct net_device *out_dev, struct net_device *route_dev)
1653	{
1654	struct mlx5_core_dev *out_mdev, *route_mdev;
1655	struct mlx5e_priv *out_priv, *route_priv;
1656
1657	out_priv = netdev_priv(dev: out_dev);
1658	out_mdev = out_priv->mdev;
1659	route_priv = netdev_priv(dev: route_dev);
1660	route_mdev = route_priv->mdev;
1661
1662	if (out_mdev->coredev_type != MLX5_COREDEV_PF)
1663	return false;
1664
1665	if (route_mdev->coredev_type != MLX5_COREDEV_VF &&
1666	route_mdev->coredev_type != MLX5_COREDEV_SF)
1667	return false;
1668
1669	return mlx5e_same_hw_devs(priv: out_priv, peer_priv: route_priv);
1670	}
1671
1672	int mlx5e_tc_query_route_vport(struct net_device *out_dev, struct net_device *route_dev, u16 *vport)
1673	{
1674	struct mlx5e_priv *out_priv, *route_priv;
1675	struct mlx5_core_dev *route_mdev;
1676	struct mlx5_devcom_comp_dev *pos;
1677	struct mlx5_eswitch *esw;
1678	u16 vhca_id;
1679	int err;
1680
1681	out_priv = netdev_priv(dev: out_dev);
1682	esw = out_priv->mdev->priv.eswitch;
1683	route_priv = netdev_priv(dev: route_dev);
1684	route_mdev = route_priv->mdev;
1685
1686	vhca_id = MLX5_CAP_GEN(route_mdev, vhca_id);
1687	err = mlx5_eswitch_vhca_id_to_vport(esw, vhca_id, vport_num: vport);
1688	if (!err)
1689	return err;
1690
1691	if (!mlx5_lag_is_active(dev: out_priv->mdev))
1692	return err;
1693
1694	rcu_read_lock();
1695	err = -ENODEV;
1696	mlx5_devcom_for_each_peer_entry_rcu(esw->devcom, esw, pos) {
1697	err = mlx5_eswitch_vhca_id_to_vport(esw, vhca_id, vport_num: vport);
1698	if (!err)
1699	break;
1700	}
1701	rcu_read_unlock();
1702
1703	return err;
1704	}
1705
1706	static int
1707	verify_attr_actions(u32 actions, struct netlink_ext_ack *extack)
1708	{
1709	if (!(actions &
1710	(MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) {
1711	NL_SET_ERR_MSG_MOD(extack, "Rule must have at least one forward/drop action");
1712	return -EOPNOTSUPP;
1713	}
1714
1715	if (!(~actions &
1716	(MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_DROP))) {
1717	NL_SET_ERR_MSG_MOD(extack, "Rule cannot support forward+drop action");
1718	return -EOPNOTSUPP;
1719	}
1720
1721	if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR &&
1722	actions & MLX5_FLOW_CONTEXT_ACTION_DROP) {
1723	NL_SET_ERR_MSG_MOD(extack, "Drop with modify header action is not supported");
1724	return -EOPNOTSUPP;
1725	}
1726
1727	return 0;
1728	}
1729
1730	static bool
1731	has_encap_dests(struct mlx5_flow_attr *attr)
1732	{
1733	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
1734	int out_index;
1735
1736	for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++)
1737	if (esw_attr->dests[out_index].flags & MLX5_ESW_DEST_ENCAP)
1738	return true;
1739
1740	return false;
1741	}
1742
1743	static int
1744	post_process_attr(struct mlx5e_tc_flow *flow,
1745	struct mlx5_flow_attr *attr,
1746	struct netlink_ext_ack *extack)
1747	{
1748	bool vf_tun;
1749	int err = 0;
1750
1751	err = verify_attr_actions(actions: attr->action, extack);
1752	if (err)
1753	goto err_out;
1754
1755	if (mlx5e_is_eswitch_flow(flow) && has_encap_dests(attr)) {
1756	err = mlx5e_tc_tun_encap_dests_set(priv: flow->priv, flow, attr, extack, vf_tun: &vf_tun);
1757	if (err)
1758	goto err_out;
1759	}
1760
1761	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
1762	err = mlx5e_tc_attach_mod_hdr(priv: flow->priv, flow, attr);
1763	if (err)
1764	goto err_out;
1765	}
1766
1767	if (attr->branch_true &&
1768	attr->branch_true->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
1769	err = mlx5e_tc_attach_mod_hdr(priv: flow->priv, flow, attr: attr->branch_true);
1770	if (err)
1771	goto err_out;
1772	}
1773
1774	if (attr->branch_false &&
1775	attr->branch_false->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
1776	err = mlx5e_tc_attach_mod_hdr(priv: flow->priv, flow, attr: attr->branch_false);
1777	if (err)
1778	goto err_out;
1779	}
1780
1781	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
1782	err = alloc_flow_attr_counter(counter_dev: get_flow_counter_dev(flow), attr);
1783	if (err)
1784	goto err_out;
1785	}
1786
1787	err_out:
1788	return err;
1789	}
1790
1791	static int
1792	mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv,
1793	struct mlx5e_tc_flow *flow,
1794	struct netlink_ext_ack *extack)
1795	{
1796	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1797	struct mlx5e_tc_flow_parse_attr *parse_attr;
1798	struct mlx5_flow_attr *attr = flow->attr;
1799	struct mlx5_esw_flow_attr *esw_attr;
1800	u32 max_prio, max_chain;
1801	int err = 0;
1802
1803	parse_attr = attr->parse_attr;
1804	esw_attr = attr->esw_attr;
1805
1806	/* We check chain range only for tc flows.
1807	* For ft flows, we checked attr->chain was originally 0 and set it to
1808	* FDB_FT_CHAIN which is outside tc range.
1809	* See mlx5e_rep_setup_ft_cb().
1810	*/
1811	max_chain = mlx5_chains_get_chain_range(esw_chains(esw));
1812	if (!mlx5e_is_ft_flow(flow) && attr->chain > max_chain) {
1813	NL_SET_ERR_MSG_MOD(extack,
1814	"Requested chain is out of supported range");
1815	err = -EOPNOTSUPP;
1816	goto err_out;
1817	}
1818
1819	max_prio = mlx5_chains_get_prio_range(esw_chains(esw));
1820	if (attr->prio > max_prio) {
1821	NL_SET_ERR_MSG_MOD(extack,
1822	"Requested priority is out of supported range");
1823	err = -EOPNOTSUPP;
1824	goto err_out;
1825	}
1826
1827	if (flow_flag_test(flow, TUN_RX)) {
1828	err = mlx5e_attach_decap_route(priv, flow);
1829	if (err)
1830	goto err_out;
1831
1832	if (!attr->chain && esw_attr->int_port &&
1833	attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
1834	/* If decap route device is internal port, change the
1835	* source vport value in reg_c0 back to uplink just in
1836	* case the rule performs goto chain > 0. If we have a miss
1837	* on chain > 0 we want the metadata regs to hold the
1838	* chain id so SW will resume handling of this packet
1839	* from the proper chain.
1840	*/
1841	u32 metadata = mlx5_eswitch_get_vport_metadata_for_set(esw,
1842	vport_num: esw_attr->in_rep->vport);
1843
1844	err = mlx5e_tc_match_to_reg_set(mdev: priv->mdev, mod_hdr_acts: &parse_attr->mod_hdr_acts,
1845	ns: MLX5_FLOW_NAMESPACE_FDB, type: VPORT_TO_REG,
1846	data: metadata);
1847	if (err)
1848	goto err_out;
1849
1850	attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1851	}
1852	}
1853
1854	if (flow_flag_test(flow, L3_TO_L2_DECAP)) {
1855	err = mlx5e_attach_decap(priv, flow, extack);
1856	if (err)
1857	goto err_out;
1858	}
1859
1860	if (netif_is_ovs_master(dev: parse_attr->filter_dev)) {
1861	struct mlx5e_tc_int_port *int_port;
1862
1863	if (attr->chain) {
1864	NL_SET_ERR_MSG_MOD(extack,
1865	"Internal port rule is only supported on chain 0");
1866	err = -EOPNOTSUPP;
1867	goto err_out;
1868	}
1869
1870	if (attr->dest_chain) {
1871	NL_SET_ERR_MSG_MOD(extack,
1872	"Internal port rule offload doesn't support goto action");
1873	err = -EOPNOTSUPP;
1874	goto err_out;
1875	}
1876
1877	int_port = mlx5e_tc_int_port_get(priv: mlx5e_get_int_port_priv(priv),
1878	ifindex: parse_attr->filter_dev->ifindex,
1879	flow_flag_test(flow, EGRESS) ?
1880	MLX5E_TC_INT_PORT_EGRESS :
1881	MLX5E_TC_INT_PORT_INGRESS);
1882	if (IS_ERR(ptr: int_port)) {
1883	err = PTR_ERR(ptr: int_port);
1884	goto err_out;
1885	}
1886
1887	esw_attr->int_port = int_port;
1888	}
1889
1890	err = post_process_attr(flow, attr, extack);
1891	if (err)
1892	goto err_out;
1893
1894	err = mlx5e_tc_act_stats_add_flow(handle: get_act_stats_handle(priv), flow);
1895	if (err)
1896	goto err_out;
1897
1898	/* we get here if one of the following takes place:
1899	* (1) there's no error
1900	* (2) there's an encap action and we don't have valid neigh
1901	*/
1902	if (flow_flag_test(flow, SLOW))
1903	flow->rule[0] = mlx5e_tc_offload_to_slow_path(esw, flow, spec: &parse_attr->spec);
1904	else
1905	flow->rule[0] = mlx5e_tc_offload_fdb_rules(esw, flow, spec: &parse_attr->spec, attr);
1906
1907	if (IS_ERR(ptr: flow->rule[0])) {
1908	err = PTR_ERR(ptr: flow->rule[0]);
1909	goto err_out;
1910	}
1911	flow_flag_set(flow, OFFLOADED);
1912
1913	return 0;
1914
1915	err_out:
1916	flow_flag_set(flow, FAILED);
1917	return err;
1918	}
1919
1920	static bool mlx5_flow_has_geneve_opt(struct mlx5e_tc_flow *flow)
1921	{
1922	struct mlx5_flow_spec *spec = &flow->attr->parse_attr->spec;
1923	void *headers_v = MLX5_ADDR_OF(fte_match_param,
1924	spec->match_value,
1925	misc_parameters_3);
1926	u32 geneve_tlv_opt_0_data = MLX5_GET(fte_match_set_misc3,
1927	headers_v,
1928	geneve_tlv_option_0_data);
1929
1930	return !!geneve_tlv_opt_0_data;
1931	}
1932
1933	static void free_branch_attr(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr)
1934	{
1935	if (!attr)
1936	return;
1937
1938	mlx5_free_flow_attr_actions(flow, attr);
1939	kvfree(addr: attr->parse_attr);
1940	kfree(objp: attr);
1941	}
1942
1943	static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv,
1944	struct mlx5e_tc_flow *flow)
1945	{
1946	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
1947	struct mlx5_flow_attr *attr = flow->attr;
1948
1949	mlx5e_put_flow_tunnel_id(flow);
1950
1951	remove_unready_flow(flow);
1952
1953	if (mlx5e_is_offloaded_flow(flow)) {
1954	if (flow_flag_test(flow, SLOW))
1955	mlx5e_tc_unoffload_from_slow_path(esw, flow);
1956	else
1957	mlx5e_tc_unoffload_fdb_rules(esw, flow, attr);
1958	}
1959	complete_all(&flow->del_hw_done);
1960
1961	if (mlx5_flow_has_geneve_opt(flow))
1962	mlx5_geneve_tlv_option_del(geneve: priv->mdev->geneve);
1963
1964	if (flow->decap_route)
1965	mlx5e_detach_decap_route(priv, flow);
1966
1967	mlx5_tc_ct_match_del(priv: get_ct_priv(priv), ct_attr: &flow->attr->ct_attr);
1968
1969	if (flow_flag_test(flow, L3_TO_L2_DECAP))
1970	mlx5e_detach_decap(priv, flow);
1971
1972	mlx5e_tc_act_stats_del_flow(handle: get_act_stats_handle(priv), flow);
1973
1974	free_flow_post_acts(flow);
1975	mlx5_free_flow_attr_actions(flow, attr);
1976
1977	kvfree(addr: attr->esw_attr->rx_tun_attr);
1978	kvfree(addr: attr->parse_attr);
1979	kfree(objp: flow->attr);
1980	}
1981
1982	struct mlx5_fc *mlx5e_tc_get_counter(struct mlx5e_tc_flow *flow)
1983	{
1984	struct mlx5_flow_attr *attr;
1985
1986	attr = list_first_entry(&flow->attrs, struct mlx5_flow_attr, list);
1987	return attr->counter;
1988	}
1989
1990	/* Iterate over tmp_list of flows attached to flow_list head. */
1991	void mlx5e_put_flow_list(struct mlx5e_priv *priv, struct list_head *flow_list)
1992	{
1993	struct mlx5e_tc_flow *flow, *tmp;
1994
1995	list_for_each_entry_safe(flow, tmp, flow_list, tmp_list)
1996	mlx5e_flow_put(priv, flow);
1997	}
1998
1999	static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow *flow,
2000	int peer_index)
2001	{
2002	struct mlx5_eswitch *esw = flow->priv->mdev->priv.eswitch;
2003	struct mlx5e_tc_flow *peer_flow;
2004	struct mlx5e_tc_flow *tmp;
2005
2006	if (!flow_flag_test(flow, ESWITCH) ||
2007	!flow_flag_test(flow, DUP))
2008	return;
2009
2010	mutex_lock(&esw->offloads.peer_mutex);
2011	list_del(entry: &flow->peer[peer_index]);
2012	mutex_unlock(lock: &esw->offloads.peer_mutex);
2013
2014	list_for_each_entry_safe(peer_flow, tmp, &flow->peer_flows, peer_flows) {
2015	if (peer_index != mlx5_get_dev_index(dev: peer_flow->priv->mdev))
2016	continue;
2017
2018	list_del(entry: &peer_flow->peer_flows);
2019	if (refcount_dec_and_test(r: &peer_flow->refcnt)) {
2020	mlx5e_tc_del_fdb_flow(priv: peer_flow->priv, flow: peer_flow);
2021	kfree(objp: peer_flow);
2022	}
2023	}
2024
2025	if (list_empty(head: &flow->peer_flows))
2026	flow_flag_clear(flow, DUP);
2027	}
2028
2029	static void mlx5e_tc_del_fdb_peers_flow(struct mlx5e_tc_flow *flow)
2030	{
2031	int i;
2032
2033	for (i = 0; i < MLX5_MAX_PORTS; i++) {
2034	if (i == mlx5_get_dev_index(dev: flow->priv->mdev))
2035	continue;
2036	mlx5e_tc_del_fdb_peer_flow(flow, peer_index: i);
2037	}
2038	}
2039
2040	static void mlx5e_tc_del_flow(struct mlx5e_priv *priv,
2041	struct mlx5e_tc_flow *flow)
2042	{
2043	if (mlx5e_is_eswitch_flow(flow)) {
2044	struct mlx5_devcom_comp_dev *devcom = flow->priv->mdev->priv.eswitch->devcom;
2045
2046	if (!mlx5_devcom_for_each_peer_begin(devcom)) {
2047	mlx5e_tc_del_fdb_flow(priv, flow);
2048	return;
2049	}
2050
2051	mlx5e_tc_del_fdb_peers_flow(flow);
2052	mlx5_devcom_for_each_peer_end(devcom);
2053	mlx5e_tc_del_fdb_flow(priv, flow);
2054	} else {
2055	mlx5e_tc_del_nic_flow(priv, flow);
2056	}
2057	}
2058
2059	static bool flow_requires_tunnel_mapping(u32 chain, struct flow_cls_offload *f)
2060	{
2061	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
2062	struct flow_action *flow_action = &rule->action;
2063	const struct flow_action_entry *act;
2064	int i;
2065
2066	if (chain)
2067	return false;
2068
2069	flow_action_for_each(i, act, flow_action) {
2070	switch (act->id) {
2071	case FLOW_ACTION_GOTO:
2072	return true;
2073	case FLOW_ACTION_SAMPLE:
2074	return true;
2075	default:
2076	continue;
2077	}
2078	}
2079
2080	return false;
2081	}
2082
2083	static int
2084	enc_opts_is_dont_care_or_full_match(struct mlx5e_priv *priv,
2085	struct flow_dissector_key_enc_opts *opts,
2086	struct netlink_ext_ack *extack,
2087	bool *dont_care)
2088	{
2089	struct geneve_opt *opt;
2090	int off = 0;
2091
2092	*dont_care = true;
2093
2094	while (opts->len > off) {
2095	opt = (struct geneve_opt *)&opts->data[off];
2096
2097	if (!(*dont_care) || opt->opt_class || opt->type ||
2098	memchr_inv(p: opt->opt_data, c: 0, size: opt->length * 4)) {
2099	*dont_care = false;
2100
2101	if (opt->opt_class != htons(U16_MAX) ||
2102	opt->type != U8_MAX) {
2103	NL_SET_ERR_MSG_MOD(extack,
2104	"Partial match of tunnel options in chain > 0 isn't supported");
2105	netdev_warn(dev: priv->netdev,
2106	format: "Partial match of tunnel options in chain > 0 isn't supported");
2107	return -EOPNOTSUPP;
2108	}
2109	}
2110
2111	off += sizeof(struct geneve_opt) + opt->length * 4;
2112	}
2113
2114	return 0;
2115	}
2116
2117	#define COPY_DISSECTOR(rule, diss_key, dst)\
2118	({ \
2119	struct flow_rule *__rule = (rule);\
2120	typeof(dst) __dst = dst;\
2121	\
2122	memcpy(__dst,\
2123	skb_flow_dissector_target(__rule->match.dissector,\
2124	diss_key,\
2125	__rule->match.key),\
2126	sizeof(*__dst));\
2127	})
2128
2129	static int mlx5e_get_flow_tunnel_id(struct mlx5e_priv *priv,
2130	struct mlx5e_tc_flow *flow,
2131	struct flow_cls_offload *f,
2132	struct net_device *filter_dev)
2133	{
2134	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
2135	struct netlink_ext_ack *extack = f->common.extack;
2136	struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts;
2137	struct flow_match_enc_opts enc_opts_match;
2138	struct tunnel_match_enc_opts tun_enc_opts;
2139	struct mlx5_rep_uplink_priv *uplink_priv;
2140	struct mlx5_flow_attr *attr = flow->attr;
2141	struct mlx5e_rep_priv *uplink_rpriv;
2142	struct tunnel_match_key tunnel_key;
2143	bool enc_opts_is_dont_care = true;
2144	u32 tun_id, enc_opts_id = 0;
2145	struct mlx5_eswitch *esw;
2146	u32 value, mask;
2147	int err;
2148
2149	esw = priv->mdev->priv.eswitch;
2150	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
2151	uplink_priv = &uplink_rpriv->uplink_priv;
2152
2153	memset(&tunnel_key, 0, sizeof(tunnel_key));
2154	COPY_DISSECTOR(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL,
2155	&tunnel_key.enc_control);
2156	if (tunnel_key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS)
2157	COPY_DISSECTOR(rule, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
2158	&tunnel_key.enc_ipv4);
2159	else
2160	COPY_DISSECTOR(rule, FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS,
2161	&tunnel_key.enc_ipv6);
2162	COPY_DISSECTOR(rule, FLOW_DISSECTOR_KEY_ENC_IP, &tunnel_key.enc_ip);
2163	COPY_DISSECTOR(rule, FLOW_DISSECTOR_KEY_ENC_PORTS,
2164	&tunnel_key.enc_tp);
2165	COPY_DISSECTOR(rule, FLOW_DISSECTOR_KEY_ENC_KEYID,
2166	&tunnel_key.enc_key_id);
2167	tunnel_key.filter_ifindex = filter_dev->ifindex;
2168
2169	err = mapping_add(ctx: uplink_priv->tunnel_mapping, data: &tunnel_key, id: &tun_id);
2170	if (err)
2171	return err;
2172
2173	flow_rule_match_enc_opts(rule, out: &enc_opts_match);
2174	err = enc_opts_is_dont_care_or_full_match(priv,
2175	opts: enc_opts_match.mask,
2176	extack,
2177	dont_care: &enc_opts_is_dont_care);
2178	if (err)
2179	goto err_enc_opts;
2180
2181	if (!enc_opts_is_dont_care) {
2182	memset(&tun_enc_opts, 0, sizeof(tun_enc_opts));
2183	memcpy(&tun_enc_opts.key, enc_opts_match.key,
2184	sizeof(*enc_opts_match.key));
2185	memcpy(&tun_enc_opts.mask, enc_opts_match.mask,
2186	sizeof(*enc_opts_match.mask));
2187
2188	err = mapping_add(ctx: uplink_priv->tunnel_enc_opts_mapping,
2189	data: &tun_enc_opts, id: &enc_opts_id);
2190	if (err)
2191	goto err_enc_opts;
2192	}
2193
2194	value = tun_id << ENC_OPTS_BITS | enc_opts_id;
2195	mask = enc_opts_id ? TUNNEL_ID_MASK :
2196	(TUNNEL_ID_MASK & ~ENC_OPTS_BITS_MASK);
2197
2198	if (attr->chain) {
2199	mlx5e_tc_match_to_reg_match(spec: &attr->parse_attr->spec,
2200	type: TUNNEL_TO_REG, val: value, mask);
2201	} else {
2202	mod_hdr_acts = &attr->parse_attr->mod_hdr_acts;
2203	err = mlx5e_tc_match_to_reg_set(mdev: priv->mdev,
2204	mod_hdr_acts, ns: MLX5_FLOW_NAMESPACE_FDB,
2205	type: TUNNEL_TO_REG, data: value);
2206	if (err)
2207	goto err_set;
2208
2209	attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2210	}
2211
2212	flow->attr->tunnel_id = value;
2213	return 0;
2214
2215	err_set:
2216	if (enc_opts_id)
2217	mapping_remove(ctx: uplink_priv->tunnel_enc_opts_mapping,
2218	id: enc_opts_id);
2219	err_enc_opts:
2220	mapping_remove(ctx: uplink_priv->tunnel_mapping, id: tun_id);
2221	return err;
2222	}
2223
2224	static void mlx5e_put_flow_tunnel_id(struct mlx5e_tc_flow *flow)
2225	{
2226	u32 enc_opts_id = flow->attr->tunnel_id & ENC_OPTS_BITS_MASK;
2227	u32 tun_id = flow->attr->tunnel_id >> ENC_OPTS_BITS;
2228	struct mlx5_rep_uplink_priv *uplink_priv;
2229	struct mlx5e_rep_priv *uplink_rpriv;
2230	struct mlx5_eswitch *esw;
2231
2232	esw = flow->priv->mdev->priv.eswitch;
2233	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
2234	uplink_priv = &uplink_rpriv->uplink_priv;
2235
2236	if (tun_id)
2237	mapping_remove(ctx: uplink_priv->tunnel_mapping, id: tun_id);
2238	if (enc_opts_id)
2239	mapping_remove(ctx: uplink_priv->tunnel_enc_opts_mapping,
2240	id: enc_opts_id);
2241	}
2242
2243	void mlx5e_tc_set_ethertype(struct mlx5_core_dev *mdev,
2244	struct flow_match_basic *match, bool outer,
2245	void *headers_c, void *headers_v)
2246	{
2247	bool ip_version_cap;
2248
2249	ip_version_cap = outer ?
2250	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2251	ft_field_support.outer_ip_version) :
2252	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
2253	ft_field_support.inner_ip_version);
2254
2255	if (ip_version_cap && match->mask->n_proto == htons(0xFFFF) &&
2256	(match->key->n_proto == htons(ETH_P_IP) ||
2257	match->key->n_proto == htons(ETH_P_IPV6))) {
2258	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_version);
2259	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
2260	match->key->n_proto == htons(ETH_P_IP) ? 4 : 6);
2261	} else {
2262	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ethertype,
2263	ntohs(match->mask->n_proto));
2264	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype,
2265	ntohs(match->key->n_proto));
2266	}
2267	}
2268
2269	u8 mlx5e_tc_get_ip_version(struct mlx5_flow_spec *spec, bool outer)
2270	{
2271	void *headers_v;
2272	u16 ethertype;
2273	u8 ip_version;
2274
2275	if (outer)
2276	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers);
2277	else
2278	headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, inner_headers);
2279
2280	ip_version = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_version);
2281	/* Return ip_version converted from ethertype anyway */
2282	if (!ip_version) {
2283	ethertype = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ethertype);
2284	if (ethertype == ETH_P_IP || ethertype == ETH_P_ARP)
2285	ip_version = 4;
2286	else if (ethertype == ETH_P_IPV6)
2287	ip_version = 6;
2288	}
2289	return ip_version;
2290	}
2291
2292	/* Tunnel device follows RFC 6040, see include/net/inet_ecn.h.
2293	* And changes inner ip_ecn depending on inner and outer ip_ecn as follows:
2294	* +---------+----------------------------------------+
2295	* |Arriving | Arriving Outer Header |
2296	* | Inner +---------+---------+---------+----------+
2297	* | Header | Not-ECT | ECT(0) | ECT(1) | CE |
2298	* +---------+---------+---------+---------+----------+
2299	* | Not-ECT | Not-ECT | Not-ECT | Not-ECT | <drop> |
2300	* | ECT(0) | ECT(0) | ECT(0) | ECT(1) | CE* |
2301	* | ECT(1) | ECT(1) | ECT(1) | ECT(1)* | CE* |
2302	* | CE | CE | CE | CE | CE |
2303	* +---------+---------+---------+---------+----------+
2304	*
2305	* Tc matches on inner after decapsulation on tunnel device, but hw offload matches
2306	* the inner ip_ecn value before hardware decap action.
2307	*
2308	* Cells marked are changed from original inner packet ip_ecn value during decap, and
2309	* so matching those values on inner ip_ecn before decap will fail.
2310	*
2311	* The following helper allows offload when inner ip_ecn won't be changed by outer ip_ecn,
2312	* except for the outer ip_ecn = CE, where in all cases inner ip_ecn will be changed to CE,
2313	* and such we can drop the inner ip_ecn=CE match.
2314	*/
2315
2316	static int mlx5e_tc_verify_tunnel_ecn(struct mlx5e_priv *priv,
2317	struct flow_cls_offload *f,
2318	bool *match_inner_ecn)
2319	{
2320	u8 outer_ecn_mask = 0, outer_ecn_key = 0, inner_ecn_mask = 0, inner_ecn_key = 0;
2321	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
2322	struct netlink_ext_ack *extack = f->common.extack;
2323	struct flow_match_ip match;
2324
2325	*match_inner_ecn = true;
2326
2327	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ENC_IP)) {
2328	flow_rule_match_enc_ip(rule, out: &match);
2329	outer_ecn_key = match.key->tos & INET_ECN_MASK;
2330	outer_ecn_mask = match.mask->tos & INET_ECN_MASK;
2331	}
2332
2333	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_IP)) {
2334	flow_rule_match_ip(rule, out: &match);
2335	inner_ecn_key = match.key->tos & INET_ECN_MASK;
2336	inner_ecn_mask = match.mask->tos & INET_ECN_MASK;
2337	}
2338
2339	if (outer_ecn_mask != 0 && outer_ecn_mask != INET_ECN_MASK) {
2340	NL_SET_ERR_MSG_MOD(extack, "Partial match on enc_tos ecn bits isn't supported");
2341	netdev_warn(dev: priv->netdev, format: "Partial match on enc_tos ecn bits isn't supported");
2342	return -EOPNOTSUPP;
2343	}
2344
2345	if (!outer_ecn_mask) {
2346	if (!inner_ecn_mask)
2347	return 0;
2348
2349	NL_SET_ERR_MSG_MOD(extack,
2350	"Matching on tos ecn bits without also matching enc_tos ecn bits isn't supported");
2351	netdev_warn(dev: priv->netdev,
2352	format: "Matching on tos ecn bits without also matching enc_tos ecn bits isn't supported");
2353	return -EOPNOTSUPP;
2354	}
2355
2356	if (inner_ecn_mask && inner_ecn_mask != INET_ECN_MASK) {
2357	NL_SET_ERR_MSG_MOD(extack,
2358	"Partial match on tos ecn bits with match on enc_tos ecn bits isn't supported");
2359	netdev_warn(dev: priv->netdev,
2360	format: "Partial match on tos ecn bits with match on enc_tos ecn bits isn't supported");
2361	return -EOPNOTSUPP;
2362	}
2363
2364	if (!inner_ecn_mask)
2365	return 0;
2366
2367	/* Both inner and outer have full mask on ecn */
2368
2369	if (outer_ecn_key == INET_ECN_ECT_1) {
2370	/* inner ecn might change by DECAP action */
2371
2372	NL_SET_ERR_MSG_MOD(extack, "Match on enc_tos ecn = ECT(1) isn't supported");
2373	netdev_warn(dev: priv->netdev, format: "Match on enc_tos ecn = ECT(1) isn't supported");
2374	return -EOPNOTSUPP;
2375	}
2376
2377	if (outer_ecn_key != INET_ECN_CE)
2378	return 0;
2379
2380	if (inner_ecn_key != INET_ECN_CE) {
2381	/* Can't happen in software, as packet ecn will be changed to CE after decap */
2382	NL_SET_ERR_MSG_MOD(extack,
2383	"Match on tos enc_tos ecn = CE while match on tos ecn != CE isn't supported");
2384	netdev_warn(dev: priv->netdev,
2385	format: "Match on tos enc_tos ecn = CE while match on tos ecn != CE isn't supported");
2386	return -EOPNOTSUPP;
2387	}
2388
2389	/* outer ecn = CE, inner ecn = CE, as decap will change inner ecn to CE in anycase,
2390	* drop match on inner ecn
2391	*/
2392	*match_inner_ecn = false;
2393
2394	return 0;
2395	}
2396
2397	static int parse_tunnel_attr(struct mlx5e_priv *priv,
2398	struct mlx5e_tc_flow *flow,
2399	struct mlx5_flow_spec *spec,
2400	struct flow_cls_offload *f,
2401	struct net_device *filter_dev,
2402	u8 *match_level,
2403	bool *match_inner)
2404	{
2405	struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev: filter_dev);
2406	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2407	struct netlink_ext_ack *extack = f->common.extack;
2408	bool needs_mapping, sets_mapping;
2409	int err;
2410
2411	if (!mlx5e_is_eswitch_flow(flow)) {
2412	NL_SET_ERR_MSG_MOD(extack, "Match on tunnel is not supported");
2413	return -EOPNOTSUPP;
2414	}
2415
2416	needs_mapping = !!flow->attr->chain;
2417	sets_mapping = flow_requires_tunnel_mapping(chain: flow->attr->chain, f);
2418	*match_inner = !needs_mapping;
2419
2420	if ((needs_mapping || sets_mapping) &&
2421	!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
2422	NL_SET_ERR_MSG_MOD(extack,
2423	"Chains on tunnel devices isn't supported without register loopback support");
2424	netdev_warn(dev: priv->netdev,
2425	format: "Chains on tunnel devices isn't supported without register loopback support");
2426	return -EOPNOTSUPP;
2427	}
2428
2429	if (!flow->attr->chain) {
2430	err = mlx5e_tc_tun_parse(filter_dev, priv, spec, f,
2431	match_level);
2432	if (err) {
2433	NL_SET_ERR_MSG_MOD(extack,
2434	"Failed to parse tunnel attributes");
2435	netdev_warn(dev: priv->netdev,
2436	format: "Failed to parse tunnel attributes");
2437	return err;
2438	}
2439
2440	/* With mpls over udp we decapsulate using packet reformat
2441	* object
2442	*/
2443	if (!netif_is_bareudp(dev: filter_dev))
2444	flow->attr->action |= MLX5_FLOW_CONTEXT_ACTION_DECAP;
2445	err = mlx5e_tc_set_attr_rx_tun(flow, spec);
2446	if (err)
2447	return err;
2448	} else if (tunnel) {
2449	struct mlx5_flow_spec *tmp_spec;
2450
2451	tmp_spec = kvzalloc(size: sizeof(*tmp_spec), GFP_KERNEL);
2452	if (!tmp_spec) {
2453	NL_SET_ERR_MSG_MOD(extack, "Failed to allocate memory for tunnel tmp spec");
2454	netdev_warn(dev: priv->netdev, format: "Failed to allocate memory for tunnel tmp spec");
2455	return -ENOMEM;
2456	}
2457	memcpy(tmp_spec, spec, sizeof(*tmp_spec));
2458
2459	err = mlx5e_tc_tun_parse(filter_dev, priv, spec: tmp_spec, f, match_level);
2460	if (err) {
2461	kvfree(addr: tmp_spec);
2462	NL_SET_ERR_MSG_MOD(extack, "Failed to parse tunnel attributes");
2463	netdev_warn(dev: priv->netdev, format: "Failed to parse tunnel attributes");
2464	return err;
2465	}
2466	err = mlx5e_tc_set_attr_rx_tun(flow, spec: tmp_spec);
2467	kvfree(addr: tmp_spec);
2468	if (err)
2469	return err;
2470	}
2471
2472	if (!needs_mapping && !sets_mapping)
2473	return 0;
2474
2475	return mlx5e_get_flow_tunnel_id(priv, flow, f, filter_dev);
2476	}
2477
2478	static void *get_match_inner_headers_criteria(struct mlx5_flow_spec *spec)
2479	{
2480	return MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
2481	inner_headers);
2482	}
2483
2484	static void *get_match_inner_headers_value(struct mlx5_flow_spec *spec)
2485	{
2486	return MLX5_ADDR_OF(fte_match_param, spec->match_value,
2487	inner_headers);
2488	}
2489
2490	static void *get_match_outer_headers_criteria(struct mlx5_flow_spec *spec)
2491	{
2492	return MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
2493	outer_headers);
2494	}
2495
2496	static void *get_match_outer_headers_value(struct mlx5_flow_spec *spec)
2497	{
2498	return MLX5_ADDR_OF(fte_match_param, spec->match_value,
2499	outer_headers);
2500	}
2501
2502	void *mlx5e_get_match_headers_value(u32 flags, struct mlx5_flow_spec *spec)
2503	{
2504	return (flags & MLX5_FLOW_CONTEXT_ACTION_DECAP) ?
2505	get_match_inner_headers_value(spec) :
2506	get_match_outer_headers_value(spec);
2507	}
2508
2509	void *mlx5e_get_match_headers_criteria(u32 flags, struct mlx5_flow_spec *spec)
2510	{
2511	return (flags & MLX5_FLOW_CONTEXT_ACTION_DECAP) ?
2512	get_match_inner_headers_criteria(spec) :
2513	get_match_outer_headers_criteria(spec);
2514	}
2515
2516	static int mlx5e_flower_parse_meta(struct net_device *filter_dev,
2517	struct flow_cls_offload *f)
2518	{
2519	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
2520	struct netlink_ext_ack *extack = f->common.extack;
2521	struct net_device *ingress_dev;
2522	struct flow_match_meta match;
2523
2524	if (!flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_META))
2525	return 0;
2526
2527	flow_rule_match_meta(rule, out: &match);
2528
2529	if (match.mask->l2_miss) {
2530	NL_SET_ERR_MSG_MOD(f->common.extack, "Can't match on \"l2_miss\"");
2531	return -EOPNOTSUPP;
2532	}
2533
2534	if (!match.mask->ingress_ifindex)
2535	return 0;
2536
2537	if (match.mask->ingress_ifindex != 0xFFFFFFFF) {
2538	NL_SET_ERR_MSG_MOD(extack, "Unsupported ingress ifindex mask");
2539	return -EOPNOTSUPP;
2540	}
2541
2542	ingress_dev = __dev_get_by_index(net: dev_net(dev: filter_dev),
2543	ifindex: match.key->ingress_ifindex);
2544	if (!ingress_dev) {
2545	NL_SET_ERR_MSG_MOD(extack,
2546	"Can't find the ingress port to match on");
2547	return -ENOENT;
2548	}
2549
2550	if (ingress_dev != filter_dev) {
2551	NL_SET_ERR_MSG_MOD(extack,
2552	"Can't match on the ingress filter port");
2553	return -EOPNOTSUPP;
2554	}
2555
2556	return 0;
2557	}
2558
2559	static bool skip_key_basic(struct net_device *filter_dev,
2560	struct flow_cls_offload *f)
2561	{
2562	/* When doing mpls over udp decap, the user needs to provide
2563	* MPLS_UC as the protocol in order to be able to match on mpls
2564	* label fields. However, the actual ethertype is IP so we want to
2565	* avoid matching on this, otherwise we'll fail the match.
2566	*/
2567	if (netif_is_bareudp(dev: filter_dev) && f->common.chain_index == 0)
2568	return true;
2569
2570	return false;
2571	}
2572
2573	static int __parse_cls_flower(struct mlx5e_priv *priv,
2574	struct mlx5e_tc_flow *flow,
2575	struct mlx5_flow_spec *spec,
2576	struct flow_cls_offload *f,
2577	struct net_device *filter_dev,
2578	u8 *inner_match_level, u8 *outer_match_level)
2579	{
2580	struct netlink_ext_ack *extack = f->common.extack;
2581	void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
2582	outer_headers);
2583	void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2584	outer_headers);
2585	void *misc_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
2586	misc_parameters);
2587	void *misc_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2588	misc_parameters);
2589	void *misc_c_3 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
2590	misc_parameters_3);
2591	void *misc_v_3 = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2592	misc_parameters_3);
2593	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
2594	struct flow_dissector *dissector = rule->match.dissector;
2595	enum fs_flow_table_type fs_type;
2596	bool match_inner_ecn = true;
2597	u16 addr_type = 0;
2598	u8 ip_proto = 0;
2599	u8 *match_level;
2600	int err;
2601
2602	fs_type = mlx5e_is_eswitch_flow(flow) ? FS_FT_FDB : FS_FT_NIC_RX;
2603	match_level = outer_match_level;
2604
2605	if (dissector->used_keys &
2606	~(BIT_ULL(FLOW_DISSECTOR_KEY_META) |
2607	BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
2608	BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
2609	BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2610	BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
2611	BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) |
2612	BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
2613	BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
2614	BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
2615	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
2616	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
2617	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
2618	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) |
2619	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
2620	BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
2621	BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
2622	BIT_ULL(FLOW_DISSECTOR_KEY_CT) |
2623	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
2624	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_OPTS) |
2625	BIT_ULL(FLOW_DISSECTOR_KEY_ICMP) |
2626	BIT_ULL(FLOW_DISSECTOR_KEY_MPLS))) {
2627	NL_SET_ERR_MSG_MOD(extack, "Unsupported key");
2628	netdev_dbg(priv->netdev, "Unsupported key used: 0x%llx\n",
2629	dissector->used_keys);
2630	return -EOPNOTSUPP;
2631	}
2632
2633	if (mlx5e_get_tc_tun(tunnel_dev: filter_dev)) {
2634	bool match_inner = false;
2635
2636	err = parse_tunnel_attr(priv, flow, spec, f, filter_dev,
2637	match_level: outer_match_level, match_inner: &match_inner);
2638	if (err)
2639	return err;
2640
2641	if (match_inner) {
2642	/* header pointers should point to the inner headers
2643	* if the packet was decapsulated already.
2644	* outer headers are set by parse_tunnel_attr.
2645	*/
2646	match_level = inner_match_level;
2647	headers_c = get_match_inner_headers_criteria(spec);
2648	headers_v = get_match_inner_headers_value(spec);
2649	}
2650
2651	err = mlx5e_tc_verify_tunnel_ecn(priv, f, match_inner_ecn: &match_inner_ecn);
2652	if (err)
2653	return err;
2654	}
2655
2656	err = mlx5e_flower_parse_meta(filter_dev, f);
2657	if (err)
2658	return err;
2659
2660	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_BASIC) &&
2661	!skip_key_basic(filter_dev, f)) {
2662	struct flow_match_basic match;
2663
2664	flow_rule_match_basic(rule, out: &match);
2665	mlx5e_tc_set_ethertype(mdev: priv->mdev, match: &match,
2666	outer: match_level == outer_match_level,
2667	headers_c, headers_v);
2668
2669	if (match.mask->n_proto)
2670	*match_level = MLX5_MATCH_L2;
2671	}
2672	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_VLAN) ||
2673	is_vlan_dev(dev: filter_dev)) {
2674	struct flow_dissector_key_vlan filter_dev_mask;
2675	struct flow_dissector_key_vlan filter_dev_key;
2676	struct flow_match_vlan match;
2677
2678	if (is_vlan_dev(dev: filter_dev)) {
2679	match.key = &filter_dev_key;
2680	match.key->vlan_id = vlan_dev_vlan_id(dev: filter_dev);
2681	match.key->vlan_tpid = vlan_dev_vlan_proto(dev: filter_dev);
2682	match.key->vlan_priority = 0;
2683	match.mask = &filter_dev_mask;
2684	memset(match.mask, 0xff, sizeof(*match.mask));
2685	match.mask->vlan_priority = 0;
2686	} else {
2687	flow_rule_match_vlan(rule, out: &match);
2688	}
2689	if (match.mask->vlan_id ||
2690	match.mask->vlan_priority ||
2691	match.mask->vlan_tpid) {
2692	if (match.key->vlan_tpid == htons(ETH_P_8021AD)) {
2693	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2694	svlan_tag, 1);
2695	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2696	svlan_tag, 1);
2697	} else {
2698	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2699	cvlan_tag, 1);
2700	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2701	cvlan_tag, 1);
2702	}
2703
2704	MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_vid,
2705	match.mask->vlan_id);
2706	MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_vid,
2707	match.key->vlan_id);
2708
2709	MLX5_SET(fte_match_set_lyr_2_4, headers_c, first_prio,
2710	match.mask->vlan_priority);
2711	MLX5_SET(fte_match_set_lyr_2_4, headers_v, first_prio,
2712	match.key->vlan_priority);
2713
2714	*match_level = MLX5_MATCH_L2;
2715
2716	if (!flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CVLAN) &&
2717	match.mask->vlan_eth_type &&
2718	MLX5_CAP_FLOWTABLE_TYPE(priv->mdev,
2719	ft_field_support.outer_second_vid,
2720	fs_type)) {
2721	MLX5_SET(fte_match_set_misc, misc_c,
2722	outer_second_cvlan_tag, 1);
2723	spec->match_criteria_enable |=
2724	MLX5_MATCH_MISC_PARAMETERS;
2725	}
2726	}
2727	} else if (*match_level != MLX5_MATCH_NONE) {
2728	/* cvlan_tag enabled in match criteria and
2729	* disabled in match value means both S & C tags
2730	* don't exist (untagged of both)
2731	*/
2732	MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1);
2733	*match_level = MLX5_MATCH_L2;
2734	}
2735
2736	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CVLAN)) {
2737	struct flow_match_vlan match;
2738
2739	flow_rule_match_cvlan(rule, out: &match);
2740	if (match.mask->vlan_id ||
2741	match.mask->vlan_priority ||
2742	match.mask->vlan_tpid) {
2743	if (!MLX5_CAP_FLOWTABLE_TYPE(priv->mdev, ft_field_support.outer_second_vid,
2744	fs_type)) {
2745	NL_SET_ERR_MSG_MOD(extack,
2746	"Matching on CVLAN is not supported");
2747	return -EOPNOTSUPP;
2748	}
2749
2750	if (match.key->vlan_tpid == htons(ETH_P_8021AD)) {
2751	MLX5_SET(fte_match_set_misc, misc_c,
2752	outer_second_svlan_tag, 1);
2753	MLX5_SET(fte_match_set_misc, misc_v,
2754	outer_second_svlan_tag, 1);
2755	} else {
2756	MLX5_SET(fte_match_set_misc, misc_c,
2757	outer_second_cvlan_tag, 1);
2758	MLX5_SET(fte_match_set_misc, misc_v,
2759	outer_second_cvlan_tag, 1);
2760	}
2761
2762	MLX5_SET(fte_match_set_misc, misc_c, outer_second_vid,
2763	match.mask->vlan_id);
2764	MLX5_SET(fte_match_set_misc, misc_v, outer_second_vid,
2765	match.key->vlan_id);
2766	MLX5_SET(fte_match_set_misc, misc_c, outer_second_prio,
2767	match.mask->vlan_priority);
2768	MLX5_SET(fte_match_set_misc, misc_v, outer_second_prio,
2769	match.key->vlan_priority);
2770
2771	*match_level = MLX5_MATCH_L2;
2772	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
2773	}
2774	}
2775
2776	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2777	struct flow_match_eth_addrs match;
2778
2779	flow_rule_match_eth_addrs(rule, out: &match);
2780	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2781	dmac_47_16),
2782	src: match.mask->dst);
2783	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2784	dmac_47_16),
2785	src: match.key->dst);
2786
2787	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2788	smac_47_16),
2789	src: match.mask->src);
2790	ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2791	smac_47_16),
2792	src: match.key->src);
2793
2794	if (!is_zero_ether_addr(addr: match.mask->src) ||
2795	!is_zero_ether_addr(addr: match.mask->dst))
2796	*match_level = MLX5_MATCH_L2;
2797	}
2798
2799	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CONTROL)) {
2800	struct flow_match_control match;
2801
2802	flow_rule_match_control(rule, out: &match);
2803	addr_type = match.key->addr_type;
2804
2805	/* the HW doesn't support frag first/later */
2806	if (match.mask->flags & FLOW_DIS_FIRST_FRAG) {
2807	NL_SET_ERR_MSG_MOD(extack, "Match on frag first/later is not supported");
2808	return -EOPNOTSUPP;
2809	}
2810
2811	if (match.mask->flags & FLOW_DIS_IS_FRAGMENT) {
2812	MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
2813	MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag,
2814	match.key->flags & FLOW_DIS_IS_FRAGMENT);
2815
2816	/* the HW doesn't need L3 inline to match on frag=no */
2817	if (!(match.key->flags & FLOW_DIS_IS_FRAGMENT))
2818	*match_level = MLX5_MATCH_L2;
2819	/* *** L2 attributes parsing up to here *** */
2820	else
2821	*match_level = MLX5_MATCH_L3;
2822	}
2823	}
2824
2825	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_BASIC)) {
2826	struct flow_match_basic match;
2827
2828	flow_rule_match_basic(rule, out: &match);
2829	ip_proto = match.key->ip_proto;
2830
2831	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
2832	match.mask->ip_proto);
2833	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
2834	match.key->ip_proto);
2835
2836	if (match.mask->ip_proto)
2837	*match_level = MLX5_MATCH_L3;
2838	}
2839
2840	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
2841	struct flow_match_ipv4_addrs match;
2842
2843	flow_rule_match_ipv4_addrs(rule, out: &match);
2844	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2845	src_ipv4_src_ipv6.ipv4_layout.ipv4),
2846	&match.mask->src, sizeof(match.mask->src));
2847	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2848	src_ipv4_src_ipv6.ipv4_layout.ipv4),
2849	&match.key->src, sizeof(match.key->src));
2850	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2851	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2852	&match.mask->dst, sizeof(match.mask->dst));
2853	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2854	dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
2855	&match.key->dst, sizeof(match.key->dst));
2856
2857	if (match.mask->src || match.mask->dst)
2858	*match_level = MLX5_MATCH_L3;
2859	}
2860
2861	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
2862	struct flow_match_ipv6_addrs match;
2863
2864	flow_rule_match_ipv6_addrs(rule, out: &match);
2865	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2866	src_ipv4_src_ipv6.ipv6_layout.ipv6),
2867	&match.mask->src, sizeof(match.mask->src));
2868	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2869	src_ipv4_src_ipv6.ipv6_layout.ipv6),
2870	&match.key->src, sizeof(match.key->src));
2871
2872	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
2873	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2874	&match.mask->dst, sizeof(match.mask->dst));
2875	memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
2876	dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
2877	&match.key->dst, sizeof(match.key->dst));
2878
2879	if (ipv6_addr_type(addr: &match.mask->src) != IPV6_ADDR_ANY ||
2880	ipv6_addr_type(addr: &match.mask->dst) != IPV6_ADDR_ANY)
2881	*match_level = MLX5_MATCH_L3;
2882	}
2883
2884	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_IP)) {
2885	struct flow_match_ip match;
2886
2887	flow_rule_match_ip(rule, out: &match);
2888	if (match_inner_ecn) {
2889	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
2890	match.mask->tos & 0x3);
2891	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
2892	match.key->tos & 0x3);
2893	}
2894
2895	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
2896	match.mask->tos >> 2);
2897	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp,
2898	match.key->tos >> 2);
2899
2900	MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit,
2901	match.mask->ttl);
2902	MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit,
2903	match.key->ttl);
2904
2905	if (match.mask->ttl &&
2906	!MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev,
2907	ft_field_support.outer_ipv4_ttl)) {
2908	NL_SET_ERR_MSG_MOD(extack,
2909	"Matching on TTL is not supported");
2910	return -EOPNOTSUPP;
2911	}
2912
2913	if (match.mask->tos || match.mask->ttl)
2914	*match_level = MLX5_MATCH_L3;
2915	}
2916
2917	/* *** L3 attributes parsing up to here *** */
2918
2919	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_PORTS)) {
2920	struct flow_match_ports match;
2921
2922	flow_rule_match_ports(rule, out: &match);
2923	switch (ip_proto) {
2924	case IPPROTO_TCP:
2925	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2926	tcp_sport, ntohs(match.mask->src));
2927	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2928	tcp_sport, ntohs(match.key->src));
2929
2930	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2931	tcp_dport, ntohs(match.mask->dst));
2932	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2933	tcp_dport, ntohs(match.key->dst));
2934	break;
2935
2936	case IPPROTO_UDP:
2937	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2938	udp_sport, ntohs(match.mask->src));
2939	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2940	udp_sport, ntohs(match.key->src));
2941
2942	MLX5_SET(fte_match_set_lyr_2_4, headers_c,
2943	udp_dport, ntohs(match.mask->dst));
2944	MLX5_SET(fte_match_set_lyr_2_4, headers_v,
2945	udp_dport, ntohs(match.key->dst));
2946	break;
2947	default:
2948	NL_SET_ERR_MSG_MOD(extack,
2949	"Only UDP and TCP transports are supported for L4 matching");
2950	netdev_err(dev: priv->netdev,
2951	format: "Only UDP and TCP transport are supported\n");
2952	return -EINVAL;
2953	}
2954
2955	if (match.mask->src || match.mask->dst)
2956	*match_level = MLX5_MATCH_L4;
2957	}
2958
2959	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_TCP)) {
2960	struct flow_match_tcp match;
2961
2962	flow_rule_match_tcp(rule, out: &match);
2963	MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
2964	ntohs(match.mask->flags));
2965	MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
2966	ntohs(match.key->flags));
2967
2968	if (match.mask->flags)
2969	*match_level = MLX5_MATCH_L4;
2970	}
2971	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ICMP)) {
2972	struct flow_match_icmp match;
2973
2974	flow_rule_match_icmp(rule, out: &match);
2975	switch (ip_proto) {
2976	case IPPROTO_ICMP:
2977	if (!(MLX5_CAP_GEN(priv->mdev, flex_parser_protocols) &
2978	MLX5_FLEX_PROTO_ICMP)) {
2979	NL_SET_ERR_MSG_MOD(extack,
2980	"Match on Flex protocols for ICMP is not supported");
2981	return -EOPNOTSUPP;
2982	}
2983	MLX5_SET(fte_match_set_misc3, misc_c_3, icmp_type,
2984	match.mask->type);
2985	MLX5_SET(fte_match_set_misc3, misc_v_3, icmp_type,
2986	match.key->type);
2987	MLX5_SET(fte_match_set_misc3, misc_c_3, icmp_code,
2988	match.mask->code);
2989	MLX5_SET(fte_match_set_misc3, misc_v_3, icmp_code,
2990	match.key->code);
2991	break;
2992	case IPPROTO_ICMPV6:
2993	if (!(MLX5_CAP_GEN(priv->mdev, flex_parser_protocols) &
2994	MLX5_FLEX_PROTO_ICMPV6)) {
2995	NL_SET_ERR_MSG_MOD(extack,
2996	"Match on Flex protocols for ICMPV6 is not supported");
2997	return -EOPNOTSUPP;
2998	}
2999	MLX5_SET(fte_match_set_misc3, misc_c_3, icmpv6_type,
3000	match.mask->type);
3001	MLX5_SET(fte_match_set_misc3, misc_v_3, icmpv6_type,
3002	match.key->type);
3003	MLX5_SET(fte_match_set_misc3, misc_c_3, icmpv6_code,
3004	match.mask->code);
3005	MLX5_SET(fte_match_set_misc3, misc_v_3, icmpv6_code,
3006	match.key->code);
3007	break;
3008	default:
3009	NL_SET_ERR_MSG_MOD(extack,
3010	"Code and type matching only with ICMP and ICMPv6");
3011	netdev_err(dev: priv->netdev,
3012	format: "Code and type matching only with ICMP and ICMPv6\n");
3013	return -EINVAL;
3014	}
3015	if (match.mask->code || match.mask->type) {
3016	*match_level = MLX5_MATCH_L4;
3017	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_3;
3018	}
3019	}
3020	/* Currently supported only for MPLS over UDP */
3021	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_MPLS) &&
3022	!netif_is_bareudp(dev: filter_dev)) {
3023	NL_SET_ERR_MSG_MOD(extack,
3024	"Matching on MPLS is supported only for MPLS over UDP");
3025	netdev_err(dev: priv->netdev,
3026	format: "Matching on MPLS is supported only for MPLS over UDP\n");
3027	return -EOPNOTSUPP;
3028	}
3029
3030	return 0;
3031	}
3032
3033	static int parse_cls_flower(struct mlx5e_priv *priv,
3034	struct mlx5e_tc_flow *flow,
3035	struct mlx5_flow_spec *spec,
3036	struct flow_cls_offload *f,
3037	struct net_device *filter_dev)
3038	{
3039	u8 inner_match_level, outer_match_level, non_tunnel_match_level;
3040	struct netlink_ext_ack *extack = f->common.extack;
3041	struct mlx5_core_dev *dev = priv->mdev;
3042	struct mlx5_eswitch *esw = dev->priv.eswitch;
3043	struct mlx5e_rep_priv *rpriv = priv->ppriv;
3044	struct mlx5_eswitch_rep *rep;
3045	bool is_eswitch_flow;
3046	int err;
3047
3048	inner_match_level = MLX5_MATCH_NONE;
3049	outer_match_level = MLX5_MATCH_NONE;
3050
3051	err = __parse_cls_flower(priv, flow, spec, f, filter_dev,
3052	inner_match_level: &inner_match_level, outer_match_level: &outer_match_level);
3053	non_tunnel_match_level = (inner_match_level == MLX5_MATCH_NONE) ?
3054	outer_match_level : inner_match_level;
3055
3056	is_eswitch_flow = mlx5e_is_eswitch_flow(flow);
3057	if (!err && is_eswitch_flow) {
3058	rep = rpriv->rep;
3059	if (rep->vport != MLX5_VPORT_UPLINK &&
3060	(esw->offloads.inline_mode != MLX5_INLINE_MODE_NONE &&
3061	esw->offloads.inline_mode < non_tunnel_match_level)) {
3062	NL_SET_ERR_MSG_MOD(extack,
3063	"Flow is not offloaded due to min inline setting");
3064	netdev_warn(dev: priv->netdev,
3065	format: "Flow is not offloaded due to min inline setting, required %d actual %d\n",
3066	non_tunnel_match_level, esw->offloads.inline_mode);
3067	return -EOPNOTSUPP;
3068	}
3069	}
3070
3071	flow->attr->inner_match_level = inner_match_level;
3072	flow->attr->outer_match_level = outer_match_level;
3073
3074
3075	return err;
3076	}
3077
3078	struct mlx5_fields {
3079	u8 field;
3080	u8 field_bsize;
3081	u32 field_mask;
3082	u32 offset;
3083	u32 match_offset;
3084	};
3085
3086	#define OFFLOAD(fw_field, field_bsize, field_mask, field, off, match_field) \
3087	{MLX5_ACTION_IN_FIELD_OUT_ ## fw_field, field_bsize, field_mask, \
3088	offsetof(struct pedit_headers, field) + (off), \
3089	MLX5_BYTE_OFF(fte_match_set_lyr_2_4, match_field)}
3090
3091	/* masked values are the same and there are no rewrites that do not have a
3092	* match.
3093	*/
3094	#define SAME_VAL_MASK(type, valp, maskp, matchvalp, matchmaskp) ({ \
3095	type matchmaskx = *(type *)(matchmaskp); \
3096	type matchvalx = *(type *)(matchvalp); \
3097	type maskx = *(type *)(maskp); \
3098	type valx = *(type *)(valp); \
3099	\
3100	(valx & maskx) == (matchvalx & matchmaskx) && !(maskx & (maskx ^ \
3101	matchmaskx)); \
3102	})
3103
3104	static bool cmp_val_mask(void *valp, void *maskp, void *matchvalp,
3105	void *matchmaskp, u8 bsize)
3106	{
3107	bool same = false;
3108
3109	switch (bsize) {
3110	case 8:
3111	same = SAME_VAL_MASK(u8, valp, maskp, matchvalp, matchmaskp);
3112	break;
3113	case 16:
3114	same = SAME_VAL_MASK(u16, valp, maskp, matchvalp, matchmaskp);
3115	break;
3116	case 32:
3117	same = SAME_VAL_MASK(u32, valp, maskp, matchvalp, matchmaskp);
3118	break;
3119	}
3120
3121	return same;
3122	}
3123
3124	static struct mlx5_fields fields[] = {
3125	OFFLOAD(DMAC_47_16, 32, U32_MAX, eth.h_dest[0], 0, dmac_47_16),
3126	OFFLOAD(DMAC_15_0, 16, U16_MAX, eth.h_dest[4], 0, dmac_15_0),
3127	OFFLOAD(SMAC_47_16, 32, U32_MAX, eth.h_source[0], 0, smac_47_16),
3128	OFFLOAD(SMAC_15_0, 16, U16_MAX, eth.h_source[4], 0, smac_15_0),
3129	OFFLOAD(ETHERTYPE, 16, U16_MAX, eth.h_proto, 0, ethertype),
3130	OFFLOAD(FIRST_VID, 16, U16_MAX, vlan.h_vlan_TCI, 0, first_vid),
3131
3132	OFFLOAD(IP_DSCP, 8, 0xfc, ip4.tos, 0, ip_dscp),
3133	OFFLOAD(IP_TTL, 8, U8_MAX, ip4.ttl, 0, ttl_hoplimit),
3134	OFFLOAD(SIPV4, 32, U32_MAX, ip4.saddr, 0, src_ipv4_src_ipv6.ipv4_layout.ipv4),
3135	OFFLOAD(DIPV4, 32, U32_MAX, ip4.daddr, 0, dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
3136
3137	OFFLOAD(SIPV6_127_96, 32, U32_MAX, ip6.saddr.s6_addr32[0], 0,
3138	src_ipv4_src_ipv6.ipv6_layout.ipv6[0]),
3139	OFFLOAD(SIPV6_95_64, 32, U32_MAX, ip6.saddr.s6_addr32[1], 0,
3140	src_ipv4_src_ipv6.ipv6_layout.ipv6[4]),
3141	OFFLOAD(SIPV6_63_32, 32, U32_MAX, ip6.saddr.s6_addr32[2], 0,
3142	src_ipv4_src_ipv6.ipv6_layout.ipv6[8]),
3143	OFFLOAD(SIPV6_31_0, 32, U32_MAX, ip6.saddr.s6_addr32[3], 0,
3144	src_ipv4_src_ipv6.ipv6_layout.ipv6[12]),
3145	OFFLOAD(DIPV6_127_96, 32, U32_MAX, ip6.daddr.s6_addr32[0], 0,
3146	dst_ipv4_dst_ipv6.ipv6_layout.ipv6[0]),
3147	OFFLOAD(DIPV6_95_64, 32, U32_MAX, ip6.daddr.s6_addr32[1], 0,
3148	dst_ipv4_dst_ipv6.ipv6_layout.ipv6[4]),
3149	OFFLOAD(DIPV6_63_32, 32, U32_MAX, ip6.daddr.s6_addr32[2], 0,
3150	dst_ipv4_dst_ipv6.ipv6_layout.ipv6[8]),
3151	OFFLOAD(DIPV6_31_0, 32, U32_MAX, ip6.daddr.s6_addr32[3], 0,
3152	dst_ipv4_dst_ipv6.ipv6_layout.ipv6[12]),
3153	OFFLOAD(IPV6_HOPLIMIT, 8, U8_MAX, ip6.hop_limit, 0, ttl_hoplimit),
3154	OFFLOAD(IP_DSCP, 16, 0x0fc0, ip6, 0, ip_dscp),
3155
3156	OFFLOAD(TCP_SPORT, 16, U16_MAX, tcp.source, 0, tcp_sport),
3157	OFFLOAD(TCP_DPORT, 16, U16_MAX, tcp.dest, 0, tcp_dport),
3158	/* in linux iphdr tcp_flags is 8 bits long */
3159	OFFLOAD(TCP_FLAGS, 8, U8_MAX, tcp.ack_seq, 5, tcp_flags),
3160
3161	OFFLOAD(UDP_SPORT, 16, U16_MAX, udp.source, 0, udp_sport),
3162	OFFLOAD(UDP_DPORT, 16, U16_MAX, udp.dest, 0, udp_dport),
3163	};
3164
3165	static u32 mask_field_get(void *mask, struct mlx5_fields *f)
3166	{
3167	switch (f->field_bsize) {
3168	case 32:
3169	return be32_to_cpu(*(__be32 *)mask) & f->field_mask;
3170	case 16:
3171	return be16_to_cpu(*(__be16 *)mask) & (u16)f->field_mask;
3172	default:
3173	return *(u8 *)mask & (u8)f->field_mask;
3174	}
3175	}
3176
3177	static void mask_field_clear(void *mask, struct mlx5_fields *f)
3178	{
3179	switch (f->field_bsize) {
3180	case 32:
3181	*(__be32 *)mask &= ~cpu_to_be32(f->field_mask);
3182	break;
3183	case 16:
3184	*(__be16 *)mask &= ~cpu_to_be16((u16)f->field_mask);
3185	break;
3186	default:
3187	*(u8 *)mask &= ~(u8)f->field_mask;
3188	break;
3189	}
3190	}
3191
3192	static int offload_pedit_fields(struct mlx5e_priv *priv,
3193	int namespace,
3194	struct mlx5e_tc_flow_parse_attr *parse_attr,
3195	u32 *action_flags,
3196	struct netlink_ext_ack *extack)
3197	{
3198	struct pedit_headers *set_masks, *add_masks, *set_vals, *add_vals;
3199	struct pedit_headers_action *hdrs = parse_attr->hdrs;
3200	void *headers_c, *headers_v, *action, *vals_p;
3201	struct mlx5e_tc_mod_hdr_acts *mod_acts;
3202	void *s_masks_p, *a_masks_p;
3203	int i, first, last, next_z;
3204	struct mlx5_fields *f;
3205	unsigned long mask;
3206	u32 s_mask, a_mask;
3207	u8 cmd;
3208
3209	mod_acts = &parse_attr->mod_hdr_acts;
3210	headers_c = mlx5e_get_match_headers_criteria(flags: *action_flags, spec: &parse_attr->spec);
3211	headers_v = mlx5e_get_match_headers_value(flags: *action_flags, spec: &parse_attr->spec);
3212
3213	set_masks = &hdrs[TCA_PEDIT_KEY_EX_CMD_SET].masks;
3214	add_masks = &hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].masks;
3215	set_vals = &hdrs[TCA_PEDIT_KEY_EX_CMD_SET].vals;
3216	add_vals = &hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].vals;
3217
3218	for (i = 0; i < ARRAY_SIZE(fields); i++) {
3219	bool skip;
3220
3221	f = &fields[i];
3222	s_masks_p = (void *)set_masks + f->offset;
3223	a_masks_p = (void *)add_masks + f->offset;
3224
3225	s_mask = mask_field_get(mask: s_masks_p, f);
3226	a_mask = mask_field_get(mask: a_masks_p, f);
3227
3228	if (!s_mask && !a_mask) /* nothing to offload here */
3229	continue;
3230
3231	if (s_mask && a_mask) {
3232	NL_SET_ERR_MSG_MOD(extack,
3233	"can't set and add to the same HW field");
3234	netdev_warn(dev: priv->netdev,
3235	format: "mlx5: can't set and add to the same HW field (%x)\n",
3236	f->field);
3237	return -EOPNOTSUPP;
3238	}
3239
3240	skip = false;
3241	if (s_mask) {
3242	void *match_mask = headers_c + f->match_offset;
3243	void *match_val = headers_v + f->match_offset;
3244
3245	cmd = MLX5_ACTION_TYPE_SET;
3246	mask = s_mask;
3247	vals_p = (void *)set_vals + f->offset;
3248	/* don't rewrite if we have a match on the same value */
3249	if (cmp_val_mask(valp: vals_p, maskp: s_masks_p, matchvalp: match_val,
3250	matchmaskp: match_mask, bsize: f->field_bsize))
3251	skip = true;
3252	/* clear to denote we consumed this field */
3253	mask_field_clear(mask: s_masks_p, f);
3254	} else {
3255	cmd = MLX5_ACTION_TYPE_ADD;
3256	mask = a_mask;
3257	vals_p = (void *)add_vals + f->offset;
3258	/* add 0 is no change */
3259	if (!mask_field_get(mask: vals_p, f))
3260	skip = true;
3261	/* clear to denote we consumed this field */
3262	mask_field_clear(mask: a_masks_p, f);
3263	}
3264	if (skip)
3265	continue;
3266
3267	first = find_first_bit(addr: &mask, size: f->field_bsize);
3268	next_z = find_next_zero_bit(addr: &mask, size: f->field_bsize, offset: first);
3269	last = find_last_bit(addr: &mask, size: f->field_bsize);
3270	if (first < next_z && next_z < last) {
3271	NL_SET_ERR_MSG_MOD(extack,
3272	"rewrite of few sub-fields isn't supported");
3273	netdev_warn(dev: priv->netdev,
3274	format: "mlx5: rewrite of few sub-fields (mask %lx) isn't offloaded\n",
3275	mask);
3276	return -EOPNOTSUPP;
3277	}
3278
3279	action = mlx5e_mod_hdr_alloc(mdev: priv->mdev, namespace, mod_hdr_acts: mod_acts);
3280	if (IS_ERR(ptr: action)) {
3281	NL_SET_ERR_MSG_MOD(extack,
3282	"too many pedit actions, can't offload");
3283	mlx5_core_warn(priv->mdev,
3284	"mlx5: parsed %d pedit actions, can't do more\n",
3285	mod_acts->num_actions);
3286	return PTR_ERR(ptr: action);
3287	}
3288
3289	MLX5_SET(set_action_in, action, action_type, cmd);
3290	MLX5_SET(set_action_in, action, field, f->field);
3291
3292	if (cmd == MLX5_ACTION_TYPE_SET) {
3293	unsigned long field_mask = f->field_mask;
3294	int start;
3295
3296	/* if field is bit sized it can start not from first bit */
3297	start = find_first_bit(addr: &field_mask, size: f->field_bsize);
3298
3299	MLX5_SET(set_action_in, action, offset, first - start);
3300	/* length is num of bits to be written, zero means length of 32 */
3301	MLX5_SET(set_action_in, action, length, (last - first + 1));
3302	}
3303
3304	if (f->field_bsize == 32)
3305	MLX5_SET(set_action_in, action, data, ntohl(*(__be32 *)vals_p) >> first);
3306	else if (f->field_bsize == 16)
3307	MLX5_SET(set_action_in, action, data, ntohs(*(__be16 *)vals_p) >> first);
3308	else if (f->field_bsize == 8)
3309	MLX5_SET(set_action_in, action, data, *(u8 *)vals_p >> first);
3310
3311	++mod_acts->num_actions;
3312	}
3313
3314	return 0;
3315	}
3316
3317	static const struct pedit_headers zero_masks = {};
3318
3319	static int verify_offload_pedit_fields(struct mlx5e_priv *priv,
3320	struct mlx5e_tc_flow_parse_attr *parse_attr,
3321	struct netlink_ext_ack *extack)
3322	{
3323	struct pedit_headers *cmd_masks;
3324	u8 cmd;
3325
3326	for (cmd = 0; cmd < __PEDIT_CMD_MAX; cmd++) {
3327	cmd_masks = &parse_attr->hdrs[cmd].masks;
3328	if (memcmp(p: cmd_masks, q: &zero_masks, size: sizeof(zero_masks))) {
3329	NL_SET_ERR_MSG_MOD(extack, "attempt to offload an unsupported field");
3330	netdev_warn(dev: priv->netdev, format: "attempt to offload an unsupported field (cmd %d)\n", cmd);
3331	print_hex_dump(KERN_WARNING, prefix_str: "mask: ", prefix_type: DUMP_PREFIX_ADDRESS,
3332	rowsize: 16, groupsize: 1, buf: cmd_masks, len: sizeof(zero_masks), ascii: true);
3333	return -EOPNOTSUPP;
3334	}
3335	}
3336
3337	return 0;
3338	}
3339
3340	static int alloc_tc_pedit_action(struct mlx5e_priv *priv, int namespace,
3341	struct mlx5e_tc_flow_parse_attr *parse_attr,
3342	u32 *action_flags,
3343	struct netlink_ext_ack *extack)
3344	{
3345	int err;
3346
3347	err = offload_pedit_fields(priv, namespace, parse_attr, action_flags, extack);
3348	if (err)
3349	goto out_dealloc_parsed_actions;
3350
3351	err = verify_offload_pedit_fields(priv, parse_attr, extack);
3352	if (err)
3353	goto out_dealloc_parsed_actions;
3354
3355	return 0;
3356
3357	out_dealloc_parsed_actions:
3358	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &parse_attr->mod_hdr_acts);
3359	return err;
3360	}
3361
3362	struct ip_ttl_word {
3363	__u8 ttl;
3364	__u8 protocol;
3365	__sum16 check;
3366	};
3367
3368	struct ipv6_hoplimit_word {
3369	__be16 payload_len;
3370	__u8 nexthdr;
3371	__u8 hop_limit;
3372	};
3373
3374	static bool
3375	is_flow_action_modify_ip_header(struct flow_action *flow_action)
3376	{
3377	const struct flow_action_entry *act;
3378	u32 mask, offset;
3379	u8 htype;
3380	int i;
3381
3382	/* For IPv4 & IPv6 header check 4 byte word,
3383	* to determine that modified fields
3384	* are NOT ttl & hop_limit only.
3385	*/
3386	flow_action_for_each(i, act, flow_action) {
3387	if (act->id != FLOW_ACTION_MANGLE &&
3388	act->id != FLOW_ACTION_ADD)
3389	continue;
3390
3391	htype = act->mangle.htype;
3392	offset = act->mangle.offset;
3393	mask = ~act->mangle.mask;
3394
3395	if (htype == FLOW_ACT_MANGLE_HDR_TYPE_IP4) {
3396	struct ip_ttl_word *ttl_word =
3397	(struct ip_ttl_word *)&mask;
3398
3399	if (offset != offsetof(struct iphdr, ttl) ||
3400	ttl_word->protocol ||
3401	ttl_word->check)
3402	return true;
3403	} else if (htype == FLOW_ACT_MANGLE_HDR_TYPE_IP6) {
3404	struct ipv6_hoplimit_word *hoplimit_word =
3405	(struct ipv6_hoplimit_word *)&mask;
3406
3407	if (offset != offsetof(struct ipv6hdr, payload_len) ||
3408	hoplimit_word->payload_len ||
3409	hoplimit_word->nexthdr)
3410	return true;
3411	}
3412	}
3413
3414	return false;
3415	}
3416
3417	static bool modify_header_match_supported(struct mlx5e_priv *priv,
3418	struct mlx5_flow_spec *spec,
3419	struct flow_action *flow_action,
3420	u32 actions,
3421	struct netlink_ext_ack *extack)
3422	{
3423	bool modify_ip_header;
3424	void *headers_c;
3425	void *headers_v;
3426	u16 ethertype;
3427	u8 ip_proto;
3428
3429	headers_c = mlx5e_get_match_headers_criteria(flags: actions, spec);
3430	headers_v = mlx5e_get_match_headers_value(flags: actions, spec);
3431	ethertype = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ethertype);
3432
3433	/* for non-IP we only re-write MACs, so we're okay */
3434	if (MLX5_GET(fte_match_set_lyr_2_4, headers_c, ip_version) == 0 &&
3435	ethertype != ETH_P_IP && ethertype != ETH_P_IPV6)
3436	goto out_ok;
3437
3438	modify_ip_header = is_flow_action_modify_ip_header(flow_action);
3439	ip_proto = MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol);
3440	if (modify_ip_header && ip_proto != IPPROTO_TCP &&
3441	ip_proto != IPPROTO_UDP && ip_proto != IPPROTO_ICMP) {
3442	NL_SET_ERR_MSG_MOD(extack,
3443	"can't offload re-write of non TCP/UDP");
3444	netdev_info(dev: priv->netdev, format: "can't offload re-write of ip proto %d\n",
3445	ip_proto);
3446	return false;
3447	}
3448
3449	out_ok:
3450	return true;
3451	}
3452
3453	static bool
3454	actions_match_supported_fdb(struct mlx5e_priv *priv,
3455	struct mlx5e_tc_flow *flow,
3456	struct netlink_ext_ack *extack)
3457	{
3458	struct mlx5_esw_flow_attr *esw_attr = flow->attr->esw_attr;
3459
3460	if (esw_attr->split_count > 0 && !mlx5_esw_has_fwd_fdb(priv->mdev)) {
3461	NL_SET_ERR_MSG_MOD(extack,
3462	"current firmware doesn't support split rule for port mirroring");
3463	netdev_warn_once(priv->netdev,
3464	"current firmware doesn't support split rule for port mirroring\n");
3465	return false;
3466	}
3467
3468	return true;
3469	}
3470
3471	static bool
3472	actions_match_supported(struct mlx5e_priv *priv,
3473	struct flow_action *flow_action,
3474	u32 actions,
3475	struct mlx5e_tc_flow_parse_attr *parse_attr,
3476	struct mlx5e_tc_flow *flow,
3477	struct netlink_ext_ack *extack)
3478	{
3479	if (actions & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR &&
3480	!modify_header_match_supported(priv, spec: &parse_attr->spec, flow_action, actions,
3481	extack))
3482	return false;
3483
3484	if (mlx5e_is_eswitch_flow(flow) &&
3485	!actions_match_supported_fdb(priv, flow, extack))
3486	return false;
3487
3488	return true;
3489	}
3490
3491	static bool same_port_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv)
3492	{
3493	return priv->mdev == peer_priv->mdev;
3494	}
3495
3496	bool mlx5e_same_hw_devs(struct mlx5e_priv *priv, struct mlx5e_priv *peer_priv)
3497	{
3498	struct mlx5_core_dev *fmdev, *pmdev;
3499	u64 fsystem_guid, psystem_guid;
3500
3501	fmdev = priv->mdev;
3502	pmdev = peer_priv->mdev;
3503
3504	fsystem_guid = mlx5_query_nic_system_image_guid(mdev: fmdev);
3505	psystem_guid = mlx5_query_nic_system_image_guid(mdev: pmdev);
3506
3507	return (fsystem_guid == psystem_guid);
3508	}
3509
3510	static int
3511	actions_prepare_mod_hdr_actions(struct mlx5e_priv *priv,
3512	struct mlx5e_tc_flow *flow,
3513	struct mlx5_flow_attr *attr,
3514	struct netlink_ext_ack *extack)
3515	{
3516	struct mlx5e_tc_flow_parse_attr *parse_attr = attr->parse_attr;
3517	struct pedit_headers_action *hdrs = parse_attr->hdrs;
3518	enum mlx5_flow_namespace_type ns_type;
3519	int err;
3520
3521	if (!hdrs[TCA_PEDIT_KEY_EX_CMD_SET].pedits &&
3522	!hdrs[TCA_PEDIT_KEY_EX_CMD_ADD].pedits)
3523	return 0;
3524
3525	ns_type = mlx5e_get_flow_namespace(flow);
3526
3527	err = alloc_tc_pedit_action(priv, namespace: ns_type, parse_attr, action_flags: &attr->action, extack);
3528	if (err)
3529	return err;
3530
3531	if (parse_attr->mod_hdr_acts.num_actions > 0)
3532	return 0;
3533
3534	/* In case all pedit actions are skipped, remove the MOD_HDR flag. */
3535	attr->action &= ~MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
3536	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &parse_attr->mod_hdr_acts);
3537
3538	if (ns_type != MLX5_FLOW_NAMESPACE_FDB)
3539	return 0;
3540
3541	if (!((attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP) ||
3542	(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH)))
3543	attr->esw_attr->split_count = 0;
3544
3545	return 0;
3546	}
3547
3548	static struct mlx5_flow_attr*
3549	mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr,
3550	enum mlx5_flow_namespace_type ns_type)
3551	{
3552	struct mlx5e_tc_flow_parse_attr *parse_attr;
3553	u32 attr_sz = ns_to_attr_sz(ns_type);
3554	struct mlx5_flow_attr *attr2;
3555
3556	attr2 = mlx5_alloc_flow_attr(type: ns_type);
3557	parse_attr = kvzalloc(size: sizeof(*parse_attr), GFP_KERNEL);
3558	if (!attr2 || !parse_attr) {
3559	kvfree(addr: parse_attr);
3560	kfree(objp: attr2);
3561	return NULL;
3562	}
3563
3564	memcpy(attr2, attr, attr_sz);
3565	INIT_LIST_HEAD(list: &attr2->list);
3566	parse_attr->filter_dev = attr->parse_attr->filter_dev;
3567	attr2->action = 0;
3568	attr2->counter = NULL;
3569	attr2->tc_act_cookies_count = 0;
3570	attr2->flags = 0;
3571	attr2->parse_attr = parse_attr;
3572	attr2->dest_chain = 0;
3573	attr2->dest_ft = NULL;
3574	attr2->act_id_restore_rule = NULL;
3575	memset(&attr2->ct_attr, 0, sizeof(attr2->ct_attr));
3576
3577	if (ns_type == MLX5_FLOW_NAMESPACE_FDB) {
3578	attr2->esw_attr->out_count = 0;
3579	attr2->esw_attr->split_count = 0;
3580	}
3581
3582	attr2->branch_true = NULL;
3583	attr2->branch_false = NULL;
3584	attr2->jumping_attr = NULL;
3585	return attr2;
3586	}
3587
3588	struct mlx5_flow_attr *
3589	mlx5e_tc_get_encap_attr(struct mlx5e_tc_flow *flow)
3590	{
3591	struct mlx5_esw_flow_attr *esw_attr;
3592	struct mlx5_flow_attr *attr;
3593	int i;
3594
3595	list_for_each_entry(attr, &flow->attrs, list) {
3596	esw_attr = attr->esw_attr;
3597	for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
3598	if (esw_attr->dests[i].flags & MLX5_ESW_DEST_ENCAP)
3599	return attr;
3600	}
3601	}
3602
3603	return NULL;
3604	}
3605
3606	void
3607	mlx5e_tc_unoffload_flow_post_acts(struct mlx5e_tc_flow *flow)
3608	{
3609	struct mlx5e_post_act *post_act = get_post_action(priv: flow->priv);
3610	struct mlx5_flow_attr *attr;
3611
3612	list_for_each_entry(attr, &flow->attrs, list) {
3613	if (list_is_last(list: &attr->list, head: &flow->attrs))
3614	break;
3615
3616	mlx5e_tc_post_act_unoffload(post_act, handle: attr->post_act_handle);
3617	}
3618	}
3619
3620	static void
3621	free_flow_post_acts(struct mlx5e_tc_flow *flow)
3622	{
3623	struct mlx5_flow_attr *attr, *tmp;
3624
3625	list_for_each_entry_safe(attr, tmp, &flow->attrs, list) {
3626	if (list_is_last(list: &attr->list, head: &flow->attrs))
3627	break;
3628
3629	mlx5_free_flow_attr_actions(flow, attr);
3630
3631	list_del(entry: &attr->list);
3632	kvfree(addr: attr->parse_attr);
3633	kfree(objp: attr);
3634	}
3635	}
3636
3637	int
3638	mlx5e_tc_offload_flow_post_acts(struct mlx5e_tc_flow *flow)
3639	{
3640	struct mlx5e_post_act *post_act = get_post_action(priv: flow->priv);
3641	struct mlx5_flow_attr *attr;
3642	int err = 0;
3643
3644	list_for_each_entry(attr, &flow->attrs, list) {
3645	if (list_is_last(list: &attr->list, head: &flow->attrs))
3646	break;
3647
3648	err = mlx5e_tc_post_act_offload(post_act, handle: attr->post_act_handle);
3649	if (err)
3650	break;
3651	}
3652
3653	return err;
3654	}
3655
3656	/* TC filter rule HW translation:
3657	*
3658	* +---------------------+
3659	* + ft prio (tc chain) +
3660	* + original match +
3661	* +---------------------+
3662	* |
3663	* | if multi table action
3664	* |
3665	* v
3666	* +---------------------+
3667	* + post act ft |<----.
3668	* + match fte id | | split on multi table action
3669	* + do actions |-----'
3670	* +---------------------+
3671	* |
3672	* |
3673	* v
3674	* Do rest of the actions after last multi table action.
3675	*/
3676	static int
3677	alloc_flow_post_acts(struct mlx5e_tc_flow *flow, struct netlink_ext_ack *extack)
3678	{
3679	struct mlx5e_post_act *post_act = get_post_action(priv: flow->priv);
3680	struct mlx5_flow_attr *attr, *next_attr = NULL;
3681	struct mlx5e_post_act_handle *handle;
3682	int err;
3683
3684	/* This is going in reverse order as needed.
3685	* The first entry is the last attribute.
3686	*/
3687	list_for_each_entry(attr, &flow->attrs, list) {
3688	if (!next_attr) {
3689	/* Set counter action on last post act rule. */
3690	attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
3691	}
3692
3693	if (next_attr && !(attr->flags & MLX5_ATTR_FLAG_TERMINATING)) {
3694	err = mlx5e_tc_act_set_next_post_act(flow, attr, next_attr);
3695	if (err)
3696	goto out_free;
3697	}
3698
3699	/* Don't add post_act rule for first attr (last in the list).
3700	* It's being handled by the caller.
3701	*/
3702	if (list_is_last(list: &attr->list, head: &flow->attrs))
3703	break;
3704
3705	err = actions_prepare_mod_hdr_actions(priv: flow->priv, flow, attr, extack);
3706	if (err)
3707	goto out_free;
3708
3709	err = post_process_attr(flow, attr, extack);
3710	if (err)
3711	goto out_free;
3712
3713	handle = mlx5e_tc_post_act_add(post_act, post_attr: attr);
3714	if (IS_ERR(ptr: handle)) {
3715	err = PTR_ERR(ptr: handle);
3716	goto out_free;
3717	}
3718
3719	attr->post_act_handle = handle;
3720
3721	if (attr->jumping_attr) {
3722	err = mlx5e_tc_act_set_next_post_act(flow, attr: attr->jumping_attr, next_attr: attr);
3723	if (err)
3724	goto out_free;
3725	}
3726
3727	next_attr = attr;
3728	}
3729
3730	if (flow_flag_test(flow, SLOW))
3731	goto out;
3732
3733	err = mlx5e_tc_offload_flow_post_acts(flow);
3734	if (err)
3735	goto out_free;
3736
3737	out:
3738	return 0;
3739
3740	out_free:
3741	free_flow_post_acts(flow);
3742	return err;
3743	}
3744
3745	static int
3746	set_branch_dest_ft(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr)
3747	{
3748	struct mlx5e_post_act *post_act = get_post_action(priv);
3749
3750	if (IS_ERR(ptr: post_act))
3751	return PTR_ERR(ptr: post_act);
3752
3753	attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
3754	attr->dest_ft = mlx5e_tc_post_act_get_ft(post_act);
3755
3756	return 0;
3757	}
3758
3759	static int
3760	alloc_branch_attr(struct mlx5e_tc_flow *flow,
3761	struct mlx5e_tc_act_branch_ctrl *cond,
3762	struct mlx5_flow_attr **cond_attr,
3763	u32 *jump_count,
3764	struct netlink_ext_ack *extack)
3765	{
3766	struct mlx5_flow_attr *attr;
3767	int err = 0;
3768
3769	*cond_attr = mlx5e_clone_flow_attr_for_post_act(attr: flow->attr,
3770	ns_type: mlx5e_get_flow_namespace(flow));
3771	if (!(*cond_attr))
3772	return -ENOMEM;
3773
3774	attr = *cond_attr;
3775
3776	switch (cond->act_id) {
3777	case FLOW_ACTION_DROP:
3778	attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP;
3779	break;
3780	case FLOW_ACTION_ACCEPT:
3781	case FLOW_ACTION_PIPE:
3782	err = set_branch_dest_ft(priv: flow->priv, attr);
3783	if (err)
3784	goto out_err;
3785	break;
3786	case FLOW_ACTION_JUMP:
3787	if (*jump_count) {
3788	NL_SET_ERR_MSG_MOD(extack, "Cannot offload flows with nested jumps");
3789	err = -EOPNOTSUPP;
3790	goto out_err;
3791	}
3792	*jump_count = cond->extval;
3793	err = set_branch_dest_ft(priv: flow->priv, attr);
3794	if (err)
3795	goto out_err;
3796	break;
3797	default:
3798	err = -EOPNOTSUPP;
3799	goto out_err;
3800	}
3801
3802	return err;
3803	out_err:
3804	kfree(objp: *cond_attr);
3805	*cond_attr = NULL;
3806	return err;
3807	}
3808
3809	static void
3810	dec_jump_count(struct flow_action_entry *act, struct mlx5e_tc_act *tc_act,
3811	struct mlx5_flow_attr *attr, struct mlx5e_priv *priv,
3812	struct mlx5e_tc_jump_state *jump_state)
3813	{
3814	if (!jump_state->jump_count)
3815	return;
3816
3817	/* Single tc action can instantiate multiple offload actions (e.g. pedit)
3818	* Jump only over a tc action
3819	*/
3820	if (act->id == jump_state->last_id && act->hw_index == jump_state->last_index)
3821	return;
3822
3823	jump_state->last_id = act->id;
3824	jump_state->last_index = act->hw_index;
3825
3826	/* nothing to do for intermediate actions */
3827	if (--jump_state->jump_count > 1)
3828	return;
3829
3830	if (jump_state->jump_count == 1) { /* last action in the jump action list */
3831
3832	/* create a new attribute after this action */
3833	jump_state->jump_target = true;
3834
3835	if (tc_act->is_terminating_action) { /* the branch ends here */
3836	attr->flags |= MLX5_ATTR_FLAG_TERMINATING;
3837	attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
3838	} else { /* the branch continues executing the rest of the actions */
3839	struct mlx5e_post_act *post_act;
3840
3841	attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
3842	post_act = get_post_action(priv);
3843	attr->dest_ft = mlx5e_tc_post_act_get_ft(post_act);
3844	}
3845	} else if (jump_state->jump_count == 0) { /* first attr after the jump action list */
3846	/* This is the post action for the jumping attribute (either red or green)
3847	* Use the stored jumping_attr to set the post act id on the jumping attribute
3848	*/
3849	attr->jumping_attr = jump_state->jumping_attr;
3850	}
3851	}
3852
3853	static int
3854	parse_branch_ctrl(struct flow_action_entry *act, struct mlx5e_tc_act *tc_act,
3855	struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr,
3856	struct mlx5e_tc_jump_state *jump_state,
3857	struct netlink_ext_ack *extack)
3858	{
3859	struct mlx5e_tc_act_branch_ctrl cond_true, cond_false;
3860	u32 jump_count = jump_state->jump_count;
3861	int err;
3862
3863	if (!tc_act->get_branch_ctrl)
3864	return 0;
3865
3866	tc_act->get_branch_ctrl(act, &cond_true, &cond_false);
3867
3868	err = alloc_branch_attr(flow, cond: &cond_true,
3869	cond_attr: &attr->branch_true, jump_count: &jump_count, extack);
3870	if (err)
3871	goto out_err;
3872
3873	if (jump_count)
3874	jump_state->jumping_attr = attr->branch_true;
3875
3876	err = alloc_branch_attr(flow, cond: &cond_false,
3877	cond_attr: &attr->branch_false, jump_count: &jump_count, extack);
3878	if (err)
3879	goto err_branch_false;
3880
3881	if (jump_count && !jump_state->jumping_attr)
3882	jump_state->jumping_attr = attr->branch_false;
3883
3884	jump_state->jump_count = jump_count;
3885
3886	/* branching action requires its own counter */
3887	attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
3888	flow_flag_set(flow, USE_ACT_STATS);
3889
3890	return 0;
3891
3892	err_branch_false:
3893	free_branch_attr(flow, attr: attr->branch_true);
3894	out_err:
3895	return err;
3896	}
3897
3898	static int
3899	parse_tc_actions(struct mlx5e_tc_act_parse_state *parse_state,
3900	struct flow_action *flow_action)
3901	{
3902	struct netlink_ext_ack *extack = parse_state->extack;
3903	struct mlx5e_tc_flow *flow = parse_state->flow;
3904	struct mlx5e_tc_jump_state jump_state = {};
3905	struct mlx5_flow_attr *attr = flow->attr;
3906	enum mlx5_flow_namespace_type ns_type;
3907	struct mlx5e_priv *priv = flow->priv;
3908	struct mlx5_flow_attr *prev_attr;
3909	struct flow_action_entry *act;
3910	struct mlx5e_tc_act *tc_act;
3911	int err, i, i_split = 0;
3912	bool is_missable;
3913
3914	ns_type = mlx5e_get_flow_namespace(flow);
3915	list_add(new: &attr->list, head: &flow->attrs);
3916
3917	flow_action_for_each(i, act, flow_action) {
3918	jump_state.jump_target = false;
3919	is_missable = false;
3920	prev_attr = attr;
3921
3922	tc_act = mlx5e_tc_act_get(act_id: act->id, ns_type);
3923	if (!tc_act) {
3924	NL_SET_ERR_MSG_MOD(extack, "Not implemented offload action");
3925	err = -EOPNOTSUPP;
3926	goto out_free_post_acts;
3927	}
3928
3929	if (tc_act->can_offload && !tc_act->can_offload(parse_state, act, i, attr)) {
3930	err = -EOPNOTSUPP;
3931	goto out_free_post_acts;
3932	}
3933
3934	err = tc_act->parse_action(parse_state, act, priv, attr);
3935	if (err)
3936	goto out_free_post_acts;
3937
3938	dec_jump_count(act, tc_act, attr, priv, jump_state: &jump_state);
3939
3940	err = parse_branch_ctrl(act, tc_act, flow, attr, jump_state: &jump_state, extack);
3941	if (err)
3942	goto out_free_post_acts;
3943
3944	parse_state->actions |= attr->action;
3945
3946	/* Split attr for multi table act if not the last act. */
3947	if (jump_state.jump_target ||
3948	(tc_act->is_multi_table_act &&
3949	tc_act->is_multi_table_act(priv, act, attr) &&
3950	i < flow_action->num_entries - 1)) {
3951	is_missable = tc_act->is_missable ? tc_act->is_missable(act) : false;
3952
3953	err = mlx5e_tc_act_post_parse(parse_state, flow_action, from: i_split, to: i, attr,
3954	ns_type);
3955	if (err)
3956	goto out_free_post_acts;
3957
3958	attr = mlx5e_clone_flow_attr_for_post_act(attr: flow->attr, ns_type);
3959	if (!attr) {
3960	err = -ENOMEM;
3961	goto out_free_post_acts;
3962	}
3963
3964	i_split = i + 1;
3965	parse_state->if_count = 0;
3966	list_add(new: &attr->list, head: &flow->attrs);
3967	}
3968
3969	if (is_missable) {
3970	/* Add counter to prev, and assign act to new (next) attr */
3971	prev_attr->action |= MLX5_FLOW_CONTEXT_ACTION_COUNT;
3972	flow_flag_set(flow, USE_ACT_STATS);
3973
3974	attr->tc_act_cookies[attr->tc_act_cookies_count++] = act->cookie;
3975	} else if (!tc_act->stats_action) {
3976	prev_attr->tc_act_cookies[prev_attr->tc_act_cookies_count++] = act->cookie;
3977	}
3978	}
3979
3980	err = mlx5e_tc_act_post_parse(parse_state, flow_action, from: i_split, to: i, attr, ns_type);
3981	if (err)
3982	goto out_free_post_acts;
3983
3984	err = alloc_flow_post_acts(flow, extack);
3985	if (err)
3986	goto out_free_post_acts;
3987
3988	return 0;
3989
3990	out_free_post_acts:
3991	free_flow_post_acts(flow);
3992
3993	return err;
3994	}
3995
3996	static int
3997	flow_action_supported(struct flow_action *flow_action,
3998	struct netlink_ext_ack *extack)
3999	{
4000	if (!flow_action_has_entries(action: flow_action)) {
4001	NL_SET_ERR_MSG_MOD(extack, "Flow action doesn't have any entries");
4002	return -EINVAL;
4003	}
4004
4005	if (!flow_action_hw_stats_check(action: flow_action, extack,
4006	allow_bit: FLOW_ACTION_HW_STATS_DELAYED_BIT)) {
4007	NL_SET_ERR_MSG_MOD(extack, "Flow action HW stats type is not supported");
4008	return -EOPNOTSUPP;
4009	}
4010
4011	return 0;
4012	}
4013
4014	static int
4015	parse_tc_nic_actions(struct mlx5e_priv *priv,
4016	struct flow_action *flow_action,
4017	struct mlx5e_tc_flow *flow,
4018	struct netlink_ext_ack *extack)
4019	{
4020	struct mlx5e_tc_act_parse_state *parse_state;
4021	struct mlx5e_tc_flow_parse_attr *parse_attr;
4022	struct mlx5_flow_attr *attr = flow->attr;
4023	int err;
4024
4025	err = flow_action_supported(flow_action, extack);
4026	if (err)
4027	return err;
4028
4029	attr->nic_attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
4030	parse_attr = attr->parse_attr;
4031	parse_state = &parse_attr->parse_state;
4032	mlx5e_tc_act_init_parse_state(parse_state, flow, flow_action, extack);
4033	parse_state->ct_priv = get_ct_priv(priv);
4034
4035	err = parse_tc_actions(parse_state, flow_action);
4036	if (err)
4037	return err;
4038
4039	err = actions_prepare_mod_hdr_actions(priv, flow, attr, extack);
4040	if (err)
4041	return err;
4042
4043	err = verify_attr_actions(actions: attr->action, extack);
4044	if (err)
4045	return err;
4046
4047	if (!actions_match_supported(priv, flow_action, actions: parse_state->actions,
4048	parse_attr, flow, extack))
4049	return -EOPNOTSUPP;
4050
4051	return 0;
4052	}
4053
4054	static bool is_merged_eswitch_vfs(struct mlx5e_priv *priv,
4055	struct net_device *peer_netdev)
4056	{
4057	struct mlx5e_priv *peer_priv;
4058
4059	peer_priv = netdev_priv(dev: peer_netdev);
4060
4061	return (MLX5_CAP_ESW(priv->mdev, merged_eswitch) &&
4062	mlx5e_eswitch_vf_rep(netdev: priv->netdev) &&
4063	mlx5e_eswitch_vf_rep(netdev: peer_netdev) &&
4064	mlx5e_same_hw_devs(priv, peer_priv));
4065	}
4066
4067	static bool same_hw_reps(struct mlx5e_priv *priv,
4068	struct net_device *peer_netdev)
4069	{
4070	struct mlx5e_priv *peer_priv;
4071
4072	peer_priv = netdev_priv(dev: peer_netdev);
4073
4074	return mlx5e_eswitch_rep(netdev: priv->netdev) &&
4075	mlx5e_eswitch_rep(netdev: peer_netdev) &&
4076	mlx5e_same_hw_devs(priv, peer_priv);
4077	}
4078
4079	static bool is_lag_dev(struct mlx5e_priv *priv,
4080	struct net_device *peer_netdev)
4081	{
4082	return ((mlx5_lag_is_sriov(dev: priv->mdev) ||
4083	mlx5_lag_is_multipath(dev: priv->mdev)) &&
4084	same_hw_reps(priv, peer_netdev));
4085	}
4086
4087	static bool is_multiport_eligible(struct mlx5e_priv *priv, struct net_device *out_dev)
4088	{
4089	return same_hw_reps(priv, peer_netdev: out_dev) && mlx5_lag_is_mpesw(dev: priv->mdev);
4090	}
4091
4092	bool mlx5e_is_valid_eswitch_fwd_dev(struct mlx5e_priv *priv,
4093	struct net_device *out_dev)
4094	{
4095	if (is_merged_eswitch_vfs(priv, peer_netdev: out_dev))
4096	return true;
4097
4098	if (is_multiport_eligible(priv, out_dev))
4099	return true;
4100
4101	if (is_lag_dev(priv, peer_netdev: out_dev))
4102	return true;
4103
4104	return mlx5e_eswitch_rep(netdev: out_dev) &&
4105	same_port_devs(priv, peer_priv: netdev_priv(dev: out_dev));
4106	}
4107
4108	int mlx5e_set_fwd_to_int_port_actions(struct mlx5e_priv *priv,
4109	struct mlx5_flow_attr *attr,
4110	int ifindex,
4111	enum mlx5e_tc_int_port_type type,
4112	u32 *action,
4113	int out_index)
4114	{
4115	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
4116	struct mlx5e_tc_int_port_priv *int_port_priv;
4117	struct mlx5e_tc_flow_parse_attr *parse_attr;
4118	struct mlx5e_tc_int_port *dest_int_port;
4119	int err;
4120
4121	parse_attr = attr->parse_attr;
4122	int_port_priv = mlx5e_get_int_port_priv(priv);
4123
4124	dest_int_port = mlx5e_tc_int_port_get(priv: int_port_priv, ifindex, type);
4125	if (IS_ERR(ptr: dest_int_port))
4126	return PTR_ERR(ptr: dest_int_port);
4127
4128	err = mlx5e_tc_match_to_reg_set(mdev: priv->mdev, mod_hdr_acts: &parse_attr->mod_hdr_acts,
4129	ns: MLX5_FLOW_NAMESPACE_FDB, type: VPORT_TO_REG,
4130	data: mlx5e_tc_int_port_get_metadata(int_port: dest_int_port));
4131	if (err) {
4132	mlx5e_tc_int_port_put(priv: int_port_priv, int_port: dest_int_port);
4133	return err;
4134	}
4135
4136	*action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
4137
4138	esw_attr->dest_int_port = dest_int_port;
4139	esw_attr->dests[out_index].flags |= MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE;
4140	esw_attr->split_count = out_index;
4141
4142	/* Forward to root fdb for matching against the new source vport */
4143	attr->dest_chain = 0;
4144
4145	return 0;
4146	}
4147
4148	static int
4149	parse_tc_fdb_actions(struct mlx5e_priv *priv,
4150	struct flow_action *flow_action,
4151	struct mlx5e_tc_flow *flow,
4152	struct netlink_ext_ack *extack)
4153	{
4154	struct mlx5e_tc_act_parse_state *parse_state;
4155	struct mlx5e_tc_flow_parse_attr *parse_attr;
4156	struct mlx5_flow_attr *attr = flow->attr;
4157	struct mlx5_esw_flow_attr *esw_attr;
4158	struct net_device *filter_dev;
4159	int err;
4160
4161	err = flow_action_supported(flow_action, extack);
4162	if (err)
4163	return err;
4164
4165	esw_attr = attr->esw_attr;
4166	parse_attr = attr->parse_attr;
4167	filter_dev = parse_attr->filter_dev;
4168	parse_state = &parse_attr->parse_state;
4169	mlx5e_tc_act_init_parse_state(parse_state, flow, flow_action, extack);
4170	parse_state->ct_priv = get_ct_priv(priv);
4171
4172	err = parse_tc_actions(parse_state, flow_action);
4173	if (err)
4174	return err;
4175
4176	/* Forward to/from internal port can only have 1 dest */
4177	if ((netif_is_ovs_master(dev: filter_dev) || esw_attr->dest_int_port) &&
4178	esw_attr->out_count > 1) {
4179	NL_SET_ERR_MSG_MOD(extack,
4180	"Rules with internal port can have only one destination");
4181	return -EOPNOTSUPP;
4182	}
4183
4184	/* Forward from tunnel/internal port to internal port is not supported */
4185	if ((mlx5e_get_tc_tun(tunnel_dev: filter_dev) || netif_is_ovs_master(dev: filter_dev)) &&
4186	esw_attr->dest_int_port) {
4187	NL_SET_ERR_MSG_MOD(extack,
4188	"Forwarding from tunnel/internal port to internal port is not supported");
4189	return -EOPNOTSUPP;
4190	}
4191
4192	err = actions_prepare_mod_hdr_actions(priv, flow, attr, extack);
4193	if (err)
4194	return err;
4195
4196	if (!actions_match_supported(priv, flow_action, actions: parse_state->actions,
4197	parse_attr, flow, extack))
4198	return -EOPNOTSUPP;
4199
4200	return 0;
4201	}
4202
4203	static void get_flags(int flags, unsigned long *flow_flags)
4204	{
4205	unsigned long __flow_flags = 0;
4206
4207	if (flags & MLX5_TC_FLAG(INGRESS))
4208	__flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_INGRESS);
4209	if (flags & MLX5_TC_FLAG(EGRESS))
4210	__flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_EGRESS);
4211
4212	if (flags & MLX5_TC_FLAG(ESW_OFFLOAD))
4213	__flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_ESWITCH);
4214	if (flags & MLX5_TC_FLAG(NIC_OFFLOAD))
4215	__flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_NIC);
4216	if (flags & MLX5_TC_FLAG(FT_OFFLOAD))
4217	__flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_FT);
4218
4219	*flow_flags = __flow_flags;
4220	}
4221
4222	static const struct rhashtable_params tc_ht_params = {
4223	.head_offset = offsetof(struct mlx5e_tc_flow, node),
4224	.key_offset = offsetof(struct mlx5e_tc_flow, cookie),
4225	.key_len = sizeof(((struct mlx5e_tc_flow *)0)->cookie),
4226	.automatic_shrinking = true,
4227	};
4228
4229	static struct rhashtable *get_tc_ht(struct mlx5e_priv *priv,
4230	unsigned long flags)
4231	{
4232	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
4233	struct mlx5e_rep_priv *rpriv;
4234
4235	if (flags & MLX5_TC_FLAG(ESW_OFFLOAD)) {
4236	rpriv = priv->ppriv;
4237	return &rpriv->tc_ht;
4238	} else /* NIC offload */
4239	return &tc->ht;
4240	}
4241
4242	static bool is_peer_flow_needed(struct mlx5e_tc_flow *flow)
4243	{
4244	struct mlx5_esw_flow_attr *esw_attr = flow->attr->esw_attr;
4245	struct mlx5_flow_attr *attr = flow->attr;
4246	bool is_rep_ingress = esw_attr->in_rep->vport != MLX5_VPORT_UPLINK &&
4247	flow_flag_test(flow, INGRESS);
4248	bool act_is_encap = !!(attr->action &
4249	MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT);
4250	bool esw_paired = mlx5_devcom_comp_is_ready(devcom: esw_attr->in_mdev->priv.eswitch->devcom);
4251
4252	if (!esw_paired)
4253	return false;
4254
4255	if ((mlx5_lag_is_sriov(dev: esw_attr->in_mdev) ||
4256	mlx5_lag_is_multipath(dev: esw_attr->in_mdev)) &&
4257	(is_rep_ingress || act_is_encap))
4258	return true;
4259
4260	if (mlx5_lag_is_mpesw(dev: esw_attr->in_mdev))
4261	return true;
4262
4263	return false;
4264	}
4265
4266	struct mlx5_flow_attr *
4267	mlx5_alloc_flow_attr(enum mlx5_flow_namespace_type type)
4268	{
4269	u32 ex_attr_size = (type == MLX5_FLOW_NAMESPACE_FDB) ?
4270	sizeof(struct mlx5_esw_flow_attr) :
4271	sizeof(struct mlx5_nic_flow_attr);
4272	struct mlx5_flow_attr *attr;
4273
4274	attr = kzalloc(size: sizeof(*attr) + ex_attr_size, GFP_KERNEL);
4275	if (!attr)
4276	return attr;
4277
4278	INIT_LIST_HEAD(list: &attr->list);
4279	return attr;
4280	}
4281
4282	static void
4283	mlx5_free_flow_attr_actions(struct mlx5e_tc_flow *flow, struct mlx5_flow_attr *attr)
4284	{
4285	struct mlx5_core_dev *counter_dev = get_flow_counter_dev(flow);
4286	struct mlx5_esw_flow_attr *esw_attr;
4287
4288	if (!attr)
4289	return;
4290
4291	if (attr->post_act_handle)
4292	mlx5e_tc_post_act_del(post_act: get_post_action(priv: flow->priv), handle: attr->post_act_handle);
4293
4294	mlx5e_tc_tun_encap_dests_unset(priv: flow->priv, flow, attr);
4295
4296	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT)
4297	mlx5_fc_destroy(dev: counter_dev, counter: attr->counter);
4298
4299	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) {
4300	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &attr->parse_attr->mod_hdr_acts);
4301	mlx5e_tc_detach_mod_hdr(priv: flow->priv, flow, attr);
4302	}
4303
4304	if (mlx5e_is_eswitch_flow(flow)) {
4305	esw_attr = attr->esw_attr;
4306
4307	if (esw_attr->int_port)
4308	mlx5e_tc_int_port_put(priv: mlx5e_get_int_port_priv(priv: flow->priv),
4309	int_port: esw_attr->int_port);
4310
4311	if (esw_attr->dest_int_port)
4312	mlx5e_tc_int_port_put(priv: mlx5e_get_int_port_priv(priv: flow->priv),
4313	int_port: esw_attr->dest_int_port);
4314	}
4315
4316	mlx5_tc_ct_delete_flow(priv: get_ct_priv(priv: flow->priv), attr);
4317
4318	free_branch_attr(flow, attr: attr->branch_true);
4319	free_branch_attr(flow, attr: attr->branch_false);
4320	}
4321
4322	static int
4323	mlx5e_alloc_flow(struct mlx5e_priv *priv, int attr_size,
4324	struct flow_cls_offload *f, unsigned long flow_flags,
4325	struct mlx5e_tc_flow_parse_attr **__parse_attr,
4326	struct mlx5e_tc_flow **__flow)
4327	{
4328	struct mlx5e_tc_flow_parse_attr *parse_attr;
4329	struct mlx5_flow_attr *attr;
4330	struct mlx5e_tc_flow *flow;
4331	int err = -ENOMEM;
4332	int out_index;
4333
4334	flow = kzalloc(size: sizeof(*flow), GFP_KERNEL);
4335	parse_attr = kvzalloc(size: sizeof(*parse_attr), GFP_KERNEL);
4336	if (!parse_attr || !flow)
4337	goto err_free;
4338
4339	flow->flags = flow_flags;
4340	flow->cookie = f->cookie;
4341	flow->priv = priv;
4342
4343	attr = mlx5_alloc_flow_attr(type: mlx5e_get_flow_namespace(flow));
4344	if (!attr)
4345	goto err_free;
4346
4347	flow->attr = attr;
4348
4349	for (out_index = 0; out_index < MLX5_MAX_FLOW_FWD_VPORTS; out_index++)
4350	INIT_LIST_HEAD(list: &flow->encaps[out_index].list);
4351	INIT_LIST_HEAD(list: &flow->hairpin);
4352	INIT_LIST_HEAD(list: &flow->l3_to_l2_reformat);
4353	INIT_LIST_HEAD(list: &flow->attrs);
4354	INIT_LIST_HEAD(list: &flow->peer_flows);
4355	refcount_set(r: &flow->refcnt, n: 1);
4356	init_completion(x: &flow->init_done);
4357	init_completion(x: &flow->del_hw_done);
4358
4359	*__flow = flow;
4360	*__parse_attr = parse_attr;
4361
4362	return 0;
4363
4364	err_free:
4365	kfree(objp: flow);
4366	kvfree(addr: parse_attr);
4367	return err;
4368	}
4369
4370	static void
4371	mlx5e_flow_attr_init(struct mlx5_flow_attr *attr,
4372	struct mlx5e_tc_flow_parse_attr *parse_attr,
4373	struct flow_cls_offload *f)
4374	{
4375	attr->parse_attr = parse_attr;
4376	attr->chain = f->common.chain_index;
4377	attr->prio = f->common.prio;
4378	}
4379
4380	static void
4381	mlx5e_flow_esw_attr_init(struct mlx5_flow_attr *attr,
4382	struct mlx5e_priv *priv,
4383	struct mlx5e_tc_flow_parse_attr *parse_attr,
4384	struct flow_cls_offload *f,
4385	struct mlx5_eswitch_rep *in_rep,
4386	struct mlx5_core_dev *in_mdev)
4387	{
4388	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
4389	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
4390
4391	mlx5e_flow_attr_init(attr, parse_attr, f);
4392
4393	esw_attr->in_rep = in_rep;
4394	esw_attr->in_mdev = in_mdev;
4395
4396	if (MLX5_CAP_ESW(esw->dev, counter_eswitch_affinity) ==
4397	MLX5_COUNTER_SOURCE_ESWITCH)
4398	esw_attr->counter_dev = in_mdev;
4399	else
4400	esw_attr->counter_dev = priv->mdev;
4401	}
4402
4403	static struct mlx5e_tc_flow *
4404	__mlx5e_add_fdb_flow(struct mlx5e_priv *priv,
4405	struct flow_cls_offload *f,
4406	unsigned long flow_flags,
4407	struct net_device *filter_dev,
4408	struct mlx5_eswitch_rep *in_rep,
4409	struct mlx5_core_dev *in_mdev)
4410	{
4411	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
4412	struct netlink_ext_ack *extack = f->common.extack;
4413	struct mlx5e_tc_flow_parse_attr *parse_attr;
4414	struct mlx5e_tc_flow *flow;
4415	int attr_size, err;
4416
4417	flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_ESWITCH);
4418	attr_size = sizeof(struct mlx5_esw_flow_attr);
4419	err = mlx5e_alloc_flow(priv, attr_size, f, flow_flags,
4420	parse_attr: &parse_attr, flow: &flow);
4421	if (err)
4422	goto out;
4423
4424	parse_attr->filter_dev = filter_dev;
4425	mlx5e_flow_esw_attr_init(attr: flow->attr,
4426	priv, parse_attr,
4427	f, in_rep, in_mdev);
4428
4429	err = parse_cls_flower(priv: flow->priv, flow, spec: &parse_attr->spec,
4430	f, filter_dev);
4431	if (err)
4432	goto err_free;
4433
4434	/* actions validation depends on parsing the ct matches first */
4435	err = mlx5_tc_ct_match_add(priv: get_ct_priv(priv), spec: &parse_attr->spec, f,
4436	ct_attr: &flow->attr->ct_attr, extack);
4437	if (err)
4438	goto err_free;
4439
4440	err = parse_tc_fdb_actions(priv, flow_action: &rule->action, flow, extack);
4441	if (err)
4442	goto err_free;
4443
4444	err = mlx5e_tc_add_fdb_flow(priv, flow, extack);
4445	complete_all(&flow->init_done);
4446	if (err) {
4447	if (!(err == -ENETUNREACH && mlx5_lag_is_multipath(dev: in_mdev)))
4448	goto err_free;
4449
4450	add_unready_flow(flow);
4451	}
4452
4453	return flow;
4454
4455	err_free:
4456	mlx5e_flow_put(priv, flow);
4457	out:
4458	return ERR_PTR(error: err);
4459	}
4460
4461	static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload *f,
4462	struct mlx5e_tc_flow *flow,
4463	unsigned long flow_flags,
4464	struct mlx5_eswitch *peer_esw)
4465	{
4466	struct mlx5e_priv *priv = flow->priv, *peer_priv;
4467	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
4468	struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr;
4469	struct mlx5e_tc_flow_parse_attr *parse_attr;
4470	int i = mlx5_get_dev_index(dev: peer_esw->dev);
4471	struct mlx5e_rep_priv *peer_urpriv;
4472	struct mlx5e_tc_flow *peer_flow;
4473	struct mlx5_core_dev *in_mdev;
4474	int err = 0;
4475
4476	peer_urpriv = mlx5_eswitch_get_uplink_priv(esw: peer_esw, rep_type: REP_ETH);
4477	peer_priv = netdev_priv(dev: peer_urpriv->netdev);
4478
4479	/* in_mdev is assigned of which the packet originated from.
4480	* So packets redirected to uplink use the same mdev of the
4481	* original flow and packets redirected from uplink use the
4482	* peer mdev.
4483	* In multiport eswitch it's a special case that we need to
4484	* keep the original mdev.
4485	*/
4486	if (attr->in_rep->vport == MLX5_VPORT_UPLINK && !mlx5_lag_is_mpesw(dev: priv->mdev))
4487	in_mdev = peer_priv->mdev;
4488	else
4489	in_mdev = priv->mdev;
4490
4491	parse_attr = flow->attr->parse_attr;
4492	peer_flow = __mlx5e_add_fdb_flow(priv: peer_priv, f, flow_flags,
4493	filter_dev: parse_attr->filter_dev,
4494	in_rep: attr->in_rep, in_mdev);
4495	if (IS_ERR(ptr: peer_flow)) {
4496	err = PTR_ERR(ptr: peer_flow);
4497	goto out;
4498	}
4499
4500	list_add_tail(new: &peer_flow->peer_flows, head: &flow->peer_flows);
4501	flow_flag_set(flow, DUP);
4502	mutex_lock(&esw->offloads.peer_mutex);
4503	list_add_tail(new: &flow->peer[i], head: &esw->offloads.peer_flows[i]);
4504	mutex_unlock(lock: &esw->offloads.peer_mutex);
4505
4506	out:
4507	return err;
4508	}
4509
4510	static int
4511	mlx5e_add_fdb_flow(struct mlx5e_priv *priv,
4512	struct flow_cls_offload *f,
4513	unsigned long flow_flags,
4514	struct net_device *filter_dev,
4515	struct mlx5e_tc_flow **__flow)
4516	{
4517	struct mlx5_devcom_comp_dev *devcom = priv->mdev->priv.eswitch->devcom, *pos;
4518	struct mlx5e_rep_priv *rpriv = priv->ppriv;
4519	struct mlx5_eswitch_rep *in_rep = rpriv->rep;
4520	struct mlx5_core_dev *in_mdev = priv->mdev;
4521	struct mlx5_eswitch *peer_esw;
4522	struct mlx5e_tc_flow *flow;
4523	int err;
4524
4525	flow = __mlx5e_add_fdb_flow(priv, f, flow_flags, filter_dev, in_rep,
4526	in_mdev);
4527	if (IS_ERR(ptr: flow))
4528	return PTR_ERR(ptr: flow);
4529
4530	if (!is_peer_flow_needed(flow)) {
4531	*__flow = flow;
4532	return 0;
4533	}
4534
4535	if (!mlx5_devcom_for_each_peer_begin(devcom)) {
4536	err = -ENODEV;
4537	goto clean_flow;
4538	}
4539
4540	mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) {
4541	err = mlx5e_tc_add_fdb_peer_flow(f, flow, flow_flags, peer_esw);
4542	if (err)
4543	goto peer_clean;
4544	}
4545
4546	mlx5_devcom_for_each_peer_end(devcom);
4547
4548	*__flow = flow;
4549	return 0;
4550
4551	peer_clean:
4552	mlx5e_tc_del_fdb_peers_flow(flow);
4553	mlx5_devcom_for_each_peer_end(devcom);
4554	clean_flow:
4555	mlx5e_tc_del_fdb_flow(priv, flow);
4556	return err;
4557	}
4558
4559	static int
4560	mlx5e_add_nic_flow(struct mlx5e_priv *priv,
4561	struct flow_cls_offload *f,
4562	unsigned long flow_flags,
4563	struct net_device *filter_dev,
4564	struct mlx5e_tc_flow **__flow)
4565	{
4566	struct flow_rule *rule = flow_cls_offload_flow_rule(flow_cmd: f);
4567	struct netlink_ext_ack *extack = f->common.extack;
4568	struct mlx5e_tc_flow_parse_attr *parse_attr;
4569	struct mlx5e_tc_flow *flow;
4570	int attr_size, err;
4571
4572	if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ignore_flow_level)) {
4573	if (!tc_cls_can_offload_and_chain0(dev: priv->netdev, common: &f->common))
4574	return -EOPNOTSUPP;
4575	} else if (!tc_can_offload_extack(dev: priv->netdev, extack: f->common.extack)) {
4576	return -EOPNOTSUPP;
4577	}
4578
4579	flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_NIC);
4580	attr_size = sizeof(struct mlx5_nic_flow_attr);
4581	err = mlx5e_alloc_flow(priv, attr_size, f, flow_flags,
4582	parse_attr: &parse_attr, flow: &flow);
4583	if (err)
4584	goto out;
4585
4586	parse_attr->filter_dev = filter_dev;
4587	mlx5e_flow_attr_init(attr: flow->attr, parse_attr, f);
4588
4589	err = parse_cls_flower(priv: flow->priv, flow, spec: &parse_attr->spec,
4590	f, filter_dev);
4591	if (err)
4592	goto err_free;
4593
4594	err = mlx5_tc_ct_match_add(priv: get_ct_priv(priv), spec: &parse_attr->spec, f,
4595	ct_attr: &flow->attr->ct_attr, extack);
4596	if (err)
4597	goto err_free;
4598
4599	err = parse_tc_nic_actions(priv, flow_action: &rule->action, flow, extack);
4600	if (err)
4601	goto err_free;
4602
4603	err = mlx5e_tc_add_nic_flow(priv, flow, extack);
4604	if (err)
4605	goto err_free;
4606
4607	flow_flag_set(flow, OFFLOADED);
4608	*__flow = flow;
4609
4610	return 0;
4611
4612	err_free:
4613	flow_flag_set(flow, FAILED);
4614	mlx5e_mod_hdr_dealloc(mod_hdr_acts: &parse_attr->mod_hdr_acts);
4615	mlx5e_flow_put(priv, flow);
4616	out:
4617	return err;
4618	}
4619
4620	static int
4621	mlx5e_tc_add_flow(struct mlx5e_priv *priv,
4622	struct flow_cls_offload *f,
4623	unsigned long flags,
4624	struct net_device *filter_dev,
4625	struct mlx5e_tc_flow **flow)
4626	{
4627	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
4628	unsigned long flow_flags;
4629	int err;
4630
4631	get_flags(flags, flow_flags: &flow_flags);
4632
4633	if (!tc_can_offload_extack(dev: priv->netdev, extack: f->common.extack))
4634	return -EOPNOTSUPP;
4635
4636	if (esw && esw->mode == MLX5_ESWITCH_OFFLOADS)
4637	err = mlx5e_add_fdb_flow(priv, f, flow_flags,
4638	filter_dev, flow: flow);
4639	else
4640	err = mlx5e_add_nic_flow(priv, f, flow_flags,
4641	filter_dev, flow: flow);
4642
4643	return err;
4644	}
4645
4646	static bool is_flow_rule_duplicate_allowed(struct net_device *dev,
4647	struct mlx5e_rep_priv *rpriv)
4648	{
4649	/* Offloaded flow rule is allowed to duplicate on non-uplink representor
4650	* sharing tc block with other slaves of a lag device. Rpriv can be NULL if this
4651	* function is called from NIC mode.
4652	*/
4653	return netif_is_lag_port(dev) && rpriv && rpriv->rep->vport != MLX5_VPORT_UPLINK;
4654	}
4655
4656	/* As IPsec and TC order is not aligned between software and hardware-offload,
4657	* either IPsec offload or TC offload, not both, is allowed for a specific interface.
4658	*/
4659	static bool is_tc_ipsec_order_check_needed(struct net_device *filter, struct mlx5e_priv *priv)
4660	{
4661	if (!IS_ENABLED(CONFIG_MLX5_EN_IPSEC))
4662	return false;
4663
4664	if (filter != priv->netdev)
4665	return false;
4666
4667	if (mlx5e_eswitch_vf_rep(netdev: priv->netdev))
4668	return false;
4669
4670	return true;
4671	}
4672
4673	static int mlx5e_tc_block_ipsec_offload(struct net_device *filter, struct mlx5e_priv *priv)
4674	{
4675	struct mlx5_core_dev *mdev = priv->mdev;
4676
4677	if (!is_tc_ipsec_order_check_needed(filter, priv))
4678	return 0;
4679
4680	if (mdev->num_block_tc)
4681	return -EBUSY;
4682
4683	mdev->num_block_ipsec++;
4684
4685	return 0;
4686	}
4687
4688	static void mlx5e_tc_unblock_ipsec_offload(struct net_device *filter, struct mlx5e_priv *priv)
4689	{
4690	if (!is_tc_ipsec_order_check_needed(filter, priv))
4691	return;
4692
4693	priv->mdev->num_block_ipsec--;
4694	}
4695
4696	int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv,
4697	struct flow_cls_offload *f, unsigned long flags)
4698	{
4699	struct netlink_ext_ack *extack = f->common.extack;
4700	struct rhashtable *tc_ht = get_tc_ht(priv, flags);
4701	struct mlx5e_rep_priv *rpriv = priv->ppriv;
4702	struct mlx5e_tc_flow *flow;
4703	int err = 0;
4704
4705	if (!mlx5_esw_hold(dev: priv->mdev))
4706	return -EBUSY;
4707
4708	err = mlx5e_tc_block_ipsec_offload(filter: dev, priv);
4709	if (err)
4710	goto esw_release;
4711
4712	mlx5_esw_get(dev: priv->mdev);
4713
4714	rcu_read_lock();
4715	flow = rhashtable_lookup(ht: tc_ht, key: &f->cookie, params: tc_ht_params);
4716	if (flow) {
4717	/* Same flow rule offloaded to non-uplink representor sharing tc block,
4718	* just return 0.
4719	*/
4720	if (is_flow_rule_duplicate_allowed(dev, rpriv) && flow->orig_dev != dev)
4721	goto rcu_unlock;
4722
4723	NL_SET_ERR_MSG_MOD(extack,
4724	"flow cookie already exists, ignoring");
4725	netdev_warn_once(priv->netdev,
4726	"flow cookie %lx already exists, ignoring\n",
4727	f->cookie);
4728	err = -EEXIST;
4729	goto rcu_unlock;
4730	}
4731	rcu_unlock:
4732	rcu_read_unlock();
4733	if (flow)
4734	goto out;
4735
4736	trace_mlx5e_configure_flower(f);
4737	err = mlx5e_tc_add_flow(priv, f, flags, filter_dev: dev, flow: &flow);
4738	if (err)
4739	goto out;
4740
4741	/* Flow rule offloaded to non-uplink representor sharing tc block,
4742	* set the flow's owner dev.
4743	*/
4744	if (is_flow_rule_duplicate_allowed(dev, rpriv))
4745	flow->orig_dev = dev;
4746
4747	err = rhashtable_lookup_insert_fast(ht: tc_ht, obj: &flow->node, params: tc_ht_params);
4748	if (err)
4749	goto err_free;
4750
4751	mlx5_esw_release(dev: priv->mdev);
4752	return 0;
4753
4754	err_free:
4755	mlx5e_flow_put(priv, flow);
4756	out:
4757	mlx5e_tc_unblock_ipsec_offload(filter: dev, priv);
4758	mlx5_esw_put(dev: priv->mdev);
4759	esw_release:
4760	mlx5_esw_release(dev: priv->mdev);
4761	return err;
4762	}
4763
4764	static bool same_flow_direction(struct mlx5e_tc_flow *flow, int flags)
4765	{
4766	bool dir_ingress = !!(flags & MLX5_TC_FLAG(INGRESS));
4767	bool dir_egress = !!(flags & MLX5_TC_FLAG(EGRESS));
4768
4769	return flow_flag_test(flow, INGRESS) == dir_ingress &&
4770	flow_flag_test(flow, EGRESS) == dir_egress;
4771	}
4772
4773	int mlx5e_delete_flower(struct net_device *dev, struct mlx5e_priv *priv,
4774	struct flow_cls_offload *f, unsigned long flags)
4775	{
4776	struct rhashtable *tc_ht = get_tc_ht(priv, flags);
4777	struct mlx5e_tc_flow *flow;
4778	int err;
4779
4780	rcu_read_lock();
4781	flow = rhashtable_lookup(ht: tc_ht, key: &f->cookie, params: tc_ht_params);
4782	if (!flow || !same_flow_direction(flow, flags)) {
4783	err = -EINVAL;
4784	goto errout;
4785	}
4786
4787	/* Only delete the flow if it doesn't have MLX5E_TC_FLOW_DELETED flag
4788	* set.
4789	*/
4790	if (flow_flag_test_and_set(flow, DELETED)) {
4791	err = -EINVAL;
4792	goto errout;
4793	}
4794	rhashtable_remove_fast(ht: tc_ht, obj: &flow->node, params: tc_ht_params);
4795	rcu_read_unlock();
4796
4797	trace_mlx5e_delete_flower(f);
4798	mlx5e_flow_put(priv, flow);
4799
4800	mlx5e_tc_unblock_ipsec_offload(filter: dev, priv);
4801	mlx5_esw_put(dev: priv->mdev);
4802	return 0;
4803
4804	errout:
4805	rcu_read_unlock();
4806	return err;
4807	}
4808
4809	int mlx5e_tc_fill_action_stats(struct mlx5e_priv *priv,
4810	struct flow_offload_action *fl_act)
4811	{
4812	return mlx5e_tc_act_stats_fill_stats(handle: get_act_stats_handle(priv), fl_act);
4813	}
4814
4815	int mlx5e_stats_flower(struct net_device *dev, struct mlx5e_priv *priv,
4816	struct flow_cls_offload *f, unsigned long flags)
4817	{
4818	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
4819	struct rhashtable *tc_ht = get_tc_ht(priv, flags);
4820	struct mlx5e_tc_flow *flow;
4821	struct mlx5_fc *counter;
4822	u64 lastuse = 0;
4823	u64 packets = 0;
4824	u64 bytes = 0;
4825	int err = 0;
4826
4827	rcu_read_lock();
4828	flow = mlx5e_flow_get(flow: rhashtable_lookup(ht: tc_ht, key: &f->cookie,
4829	params: tc_ht_params));
4830	rcu_read_unlock();
4831	if (IS_ERR(ptr: flow))
4832	return PTR_ERR(ptr: flow);
4833
4834	if (!same_flow_direction(flow, flags)) {
4835	err = -EINVAL;
4836	goto errout;
4837	}
4838
4839	if (mlx5e_is_offloaded_flow(flow)) {
4840	if (flow_flag_test(flow, USE_ACT_STATS)) {
4841	f->use_act_stats = true;
4842	} else {
4843	counter = mlx5e_tc_get_counter(flow);
4844	if (!counter)
4845	goto errout;
4846
4847	mlx5_fc_query_cached(counter, bytes: &bytes, packets: &packets, lastuse: &lastuse);
4848	}
4849	}
4850
4851	/* Under multipath it's possible for one rule to be currently
4852	* un-offloaded while the other rule is offloaded.
4853	*/
4854	if (esw && !mlx5_devcom_for_each_peer_begin(devcom: esw->devcom))
4855	goto out;
4856
4857	if (flow_flag_test(flow, DUP)) {
4858	struct mlx5e_tc_flow *peer_flow;
4859
4860	list_for_each_entry(peer_flow, &flow->peer_flows, peer_flows) {
4861	u64 packets2;
4862	u64 lastuse2;
4863	u64 bytes2;
4864
4865	if (!flow_flag_test(peer_flow, OFFLOADED))
4866	continue;
4867	if (flow_flag_test(flow, USE_ACT_STATS)) {
4868	f->use_act_stats = true;
4869	break;
4870	}
4871
4872	counter = mlx5e_tc_get_counter(flow: peer_flow);
4873	if (!counter)
4874	goto no_peer_counter;
4875	mlx5_fc_query_cached(counter, bytes: &bytes2, packets: &packets2,
4876	lastuse: &lastuse2);
4877
4878	bytes += bytes2;
4879	packets += packets2;
4880	lastuse = max_t(u64, lastuse, lastuse2);
4881	}
4882	}
4883
4884	no_peer_counter:
4885	if (esw)
4886	mlx5_devcom_for_each_peer_end(devcom: esw->devcom);
4887	out:
4888	flow_stats_update(flow_stats: &f->stats, bytes, pkts: packets, drops: 0, lastused: lastuse,
4889	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
4890	trace_mlx5e_stats_flower(f);
4891	errout:
4892	mlx5e_flow_put(priv, flow);
4893	return err;
4894	}
4895
4896	static int apply_police_params(struct mlx5e_priv *priv, u64 rate,
4897	struct netlink_ext_ack *extack)
4898	{
4899	struct mlx5e_rep_priv *rpriv = priv->ppriv;
4900	struct mlx5_eswitch *esw;
4901	u32 rate_mbps = 0;
4902	u16 vport_num;
4903	int err;
4904
4905	vport_num = rpriv->rep->vport;
4906	if (vport_num >= MLX5_VPORT_ECPF) {
4907	NL_SET_ERR_MSG_MOD(extack,
4908	"Ingress rate limit is supported only for Eswitch ports connected to VFs");
4909	return -EOPNOTSUPP;
4910	}
4911
4912	esw = priv->mdev->priv.eswitch;
4913	/* rate is given in bytes/sec.
4914	* First convert to bits/sec and then round to the nearest mbit/secs.
4915	* mbit means million bits.
4916	* Moreover, if rate is non zero we choose to configure to a minimum of
4917	* 1 mbit/sec.
4918	*/
4919	if (rate) {
4920	rate = (rate * BITS_PER_BYTE) + 500000;
4921	do_div(rate, 1000000);
4922	rate_mbps = max_t(u32, rate, 1);
4923	}
4924
4925	err = mlx5_esw_qos_modify_vport_rate(esw, vport_num, rate_mbps);
4926	if (err)
4927	NL_SET_ERR_MSG_MOD(extack, "failed applying action to hardware");
4928
4929	return err;
4930	}
4931
4932	static int
4933	tc_matchall_police_validate(const struct flow_action *action,
4934	const struct flow_action_entry *act,
4935	struct netlink_ext_ack *extack)
4936	{
4937	if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE) {
4938	NL_SET_ERR_MSG_MOD(extack,
4939	"Offload not supported when conform action is not continue");
4940	return -EOPNOTSUPP;
4941	}
4942
4943	if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
4944	NL_SET_ERR_MSG_MOD(extack,
4945	"Offload not supported when exceed action is not drop");
4946	return -EOPNOTSUPP;
4947	}
4948
4949	if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
4950	!flow_action_is_last_entry(action, entry: act)) {
4951	NL_SET_ERR_MSG_MOD(extack,
4952	"Offload not supported when conform action is ok, but action is not last");
4953	return -EOPNOTSUPP;
4954	}
4955
4956	if (act->police.peakrate_bytes_ps ||
4957	act->police.avrate || act->police.overhead) {
4958	NL_SET_ERR_MSG_MOD(extack,
4959	"Offload not supported when peakrate/avrate/overhead is configured");
4960	return -EOPNOTSUPP;
4961	}
4962
4963	return 0;
4964	}
4965
4966	static int scan_tc_matchall_fdb_actions(struct mlx5e_priv *priv,
4967	struct flow_action *flow_action,
4968	struct netlink_ext_ack *extack)
4969	{
4970	struct mlx5e_rep_priv *rpriv = priv->ppriv;
4971	const struct flow_action_entry *act;
4972	int err;
4973	int i;
4974
4975	if (!flow_action_has_entries(action: flow_action)) {
4976	NL_SET_ERR_MSG_MOD(extack, "matchall called with no action");
4977	return -EINVAL;
4978	}
4979
4980	if (!flow_offload_has_one_action(action: flow_action)) {
4981	NL_SET_ERR_MSG_MOD(extack, "matchall policing support only a single action");
4982	return -EOPNOTSUPP;
4983	}
4984
4985	if (!flow_action_basic_hw_stats_check(action: flow_action, extack)) {
4986	NL_SET_ERR_MSG_MOD(extack, "Flow action HW stats type is not supported");
4987	return -EOPNOTSUPP;
4988	}
4989
4990	flow_action_for_each(i, act, flow_action) {
4991	switch (act->id) {
4992	case FLOW_ACTION_POLICE:
4993	err = tc_matchall_police_validate(action: flow_action, act, extack);
4994	if (err)
4995	return err;
4996
4997	err = apply_police_params(priv, rate: act->police.rate_bytes_ps, extack);
4998	if (err)
4999	return err;
5000
5001	mlx5e_stats_copy_rep_stats(vf_vport: &rpriv->prev_vf_vport_stats,
5002	rep_stats: &priv->stats.rep_stats);
5003	break;
5004	default:
5005	NL_SET_ERR_MSG_MOD(extack, "mlx5 supports only police action for matchall");
5006	return -EOPNOTSUPP;
5007	}
5008	}
5009
5010	return 0;
5011	}
5012
5013	int mlx5e_tc_configure_matchall(struct mlx5e_priv *priv,
5014	struct tc_cls_matchall_offload *ma)
5015	{
5016	struct netlink_ext_ack *extack = ma->common.extack;
5017
5018	if (ma->common.prio != 1) {
5019	NL_SET_ERR_MSG_MOD(extack, "only priority 1 is supported");
5020	return -EINVAL;
5021	}
5022
5023	return scan_tc_matchall_fdb_actions(priv, flow_action: &ma->rule->action, extack);
5024	}
5025
5026	int mlx5e_tc_delete_matchall(struct mlx5e_priv *priv,
5027	struct tc_cls_matchall_offload *ma)
5028	{
5029	struct netlink_ext_ack *extack = ma->common.extack;
5030
5031	return apply_police_params(priv, rate: 0, extack);
5032	}
5033
5034	static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv,
5035	struct mlx5e_priv *peer_priv)
5036	{
5037	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
5038	struct mlx5_core_dev *peer_mdev = peer_priv->mdev;
5039	struct mlx5e_hairpin_entry *hpe, *tmp;
5040	LIST_HEAD(init_wait_list);
5041	u16 peer_vhca_id;
5042	int bkt;
5043
5044	if (!mlx5e_same_hw_devs(priv, peer_priv))
5045	return;
5046
5047	peer_vhca_id = MLX5_CAP_GEN(peer_mdev, vhca_id);
5048
5049	mutex_lock(&tc->hairpin_tbl_lock);
5050	hash_for_each(tc->hairpin_tbl, bkt, hpe, hairpin_hlist)
5051	if (refcount_inc_not_zero(r: &hpe->refcnt))
5052	list_add(new: &hpe->dead_peer_wait_list, head: &init_wait_list);
5053	mutex_unlock(lock: &tc->hairpin_tbl_lock);
5054
5055	list_for_each_entry_safe(hpe, tmp, &init_wait_list, dead_peer_wait_list) {
5056	wait_for_completion(&hpe->res_ready);
5057	if (!IS_ERR_OR_NULL(ptr: hpe->hp) && hpe->peer_vhca_id == peer_vhca_id)
5058	mlx5_core_hairpin_clear_dead_peer(hp: hpe->hp->pair);
5059
5060	mlx5e_hairpin_put(priv, hpe);
5061	}
5062	}
5063
5064	static int mlx5e_tc_netdev_event(struct notifier_block *this,
5065	unsigned long event, void *ptr)
5066	{
5067	struct net_device *ndev = netdev_notifier_info_to_dev(info: ptr);
5068	struct mlx5e_priv *peer_priv;
5069	struct mlx5e_tc_table *tc;
5070	struct mlx5e_priv *priv;
5071
5072	if (ndev->netdev_ops != &mlx5e_netdev_ops ||
5073	event != NETDEV_UNREGISTER ||
5074	ndev->reg_state == NETREG_REGISTERED)
5075	return NOTIFY_DONE;
5076
5077	tc = container_of(this, struct mlx5e_tc_table, netdevice_nb);
5078	priv = tc->priv;
5079	peer_priv = netdev_priv(dev: ndev);
5080	if (priv == peer_priv ||
5081	!(priv->netdev->features & NETIF_F_HW_TC))
5082	return NOTIFY_DONE;
5083
5084	mlx5e_tc_hairpin_update_dead_peer(priv, peer_priv);
5085
5086	return NOTIFY_DONE;
5087	}
5088
5089	static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv)
5090	{
5091	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
5092	struct mlx5_flow_table **ft = &tc->miss_t;
5093	struct mlx5_flow_table_attr ft_attr = {};
5094	struct mlx5_flow_namespace *ns;
5095	int err = 0;
5096
5097	ft_attr.max_fte = 1;
5098	ft_attr.autogroup.max_num_groups = 1;
5099	ft_attr.level = MLX5E_TC_MISS_LEVEL;
5100	ft_attr.prio = 0;
5101	ns = mlx5_get_flow_namespace(dev: priv->mdev, type: MLX5_FLOW_NAMESPACE_KERNEL);
5102
5103	*ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr: &ft_attr);
5104	if (IS_ERR(ptr: *ft)) {
5105	err = PTR_ERR(ptr: *ft);
5106	netdev_err(dev: priv->netdev, format: "failed to create tc nic miss table err=%d\n", err);
5107	}
5108
5109	return err;
5110	}
5111
5112	static void mlx5e_tc_nic_destroy_miss_table(struct mlx5e_priv *priv)
5113	{
5114	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
5115
5116	mlx5_destroy_flow_table(ft: tc->miss_t);
5117	}
5118
5119	int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
5120	{
5121	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
5122	struct mlx5_core_dev *dev = priv->mdev;
5123	struct mapping_ctx *chains_mapping;
5124	struct mlx5_chains_attr attr = {};
5125	u64 mapping_id;
5126	int err;
5127
5128	mlx5e_mod_hdr_tbl_init(tbl: &tc->mod_hdr);
5129	mutex_init(&tc->t_lock);
5130	mutex_init(&tc->hairpin_tbl_lock);
5131	hash_init(tc->hairpin_tbl);
5132	tc->priv = priv;
5133
5134	err = rhashtable_init(ht: &tc->ht, params: &tc_ht_params);
5135	if (err)
5136	return err;
5137
5138	lockdep_set_class(&tc->ht.mutex, &tc_ht_lock_key);
5139	lockdep_init_map(lock: &tc->ht.run_work.lockdep_map, name: "tc_ht_wq_key", key: &tc_ht_wq_key, subclass: 0);
5140
5141	mapping_id = mlx5_query_nic_system_image_guid(mdev: dev);
5142
5143	chains_mapping = mapping_create_for_id(id: mapping_id, type: MAPPING_TYPE_CHAIN,
5144	data_size: sizeof(struct mlx5_mapped_obj),
5145	MLX5E_TC_TABLE_CHAIN_TAG_MASK, delayed_removal: true);
5146
5147	if (IS_ERR(ptr: chains_mapping)) {
5148	err = PTR_ERR(ptr: chains_mapping);
5149	goto err_mapping;
5150	}
5151	tc->mapping = chains_mapping;
5152
5153	err = mlx5e_tc_nic_create_miss_table(priv);
5154	if (err)
5155	goto err_chains;
5156
5157	if (MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ignore_flow_level))
5158	attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED |
5159	MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
5160	attr.ns = MLX5_FLOW_NAMESPACE_KERNEL;
5161	attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS;
5162	attr.default_ft = tc->miss_t;
5163	attr.mapping = chains_mapping;
5164	attr.fs_base_prio = MLX5E_TC_PRIO;
5165
5166	tc->chains = mlx5_chains_create(dev, attr: &attr);
5167	if (IS_ERR(ptr: tc->chains)) {
5168	err = PTR_ERR(ptr: tc->chains);
5169	goto err_miss;
5170	}
5171
5172	mlx5_chains_print_info(chains: tc->chains);
5173
5174	tc->post_act = mlx5e_tc_post_act_init(priv, chains: tc->chains, ns_type: MLX5_FLOW_NAMESPACE_KERNEL);
5175	tc->ct = mlx5_tc_ct_init(priv, chains: tc->chains, mod_hdr: &tc->mod_hdr,
5176	ns_type: MLX5_FLOW_NAMESPACE_KERNEL, post_act: tc->post_act);
5177
5178	tc->netdevice_nb.notifier_call = mlx5e_tc_netdev_event;
5179	err = register_netdevice_notifier_dev_net(dev: priv->netdev,
5180	nb: &tc->netdevice_nb,
5181	nn: &tc->netdevice_nn);
5182	if (err) {
5183	tc->netdevice_nb.notifier_call = NULL;
5184	mlx5_core_warn(priv->mdev, "Failed to register netdev notifier\n");
5185	goto err_reg;
5186	}
5187
5188	mlx5e_tc_debugfs_init(tc, dfs_root: mlx5e_fs_get_debugfs_root(fs: priv->fs));
5189
5190	tc->action_stats_handle = mlx5e_tc_act_stats_create();
5191	if (IS_ERR(ptr: tc->action_stats_handle)) {
5192	err = PTR_ERR(ptr: tc->action_stats_handle);
5193	goto err_act_stats;
5194	}
5195
5196	return 0;
5197
5198	err_act_stats:
5199	unregister_netdevice_notifier_dev_net(dev: priv->netdev,
5200	nb: &tc->netdevice_nb,
5201	nn: &tc->netdevice_nn);
5202	err_reg:
5203	mlx5_tc_ct_clean(ct_priv: tc->ct);
5204	mlx5e_tc_post_act_destroy(post_act: tc->post_act);
5205	mlx5_chains_destroy(chains: tc->chains);
5206	err_miss:
5207	mlx5e_tc_nic_destroy_miss_table(priv);
5208	err_chains:
5209	mapping_destroy(ctx: chains_mapping);
5210	err_mapping:
5211	rhashtable_destroy(ht: &tc->ht);
5212	return err;
5213	}
5214
5215	static void _mlx5e_tc_del_flow(void *ptr, void *arg)
5216	{
5217	struct mlx5e_tc_flow *flow = ptr;
5218	struct mlx5e_priv *priv = flow->priv;
5219
5220	mlx5e_tc_del_flow(priv, flow);
5221	kfree(objp: flow);
5222	}
5223
5224	void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv)
5225	{
5226	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(fs: priv->fs);
5227
5228	debugfs_remove_recursive(dentry: tc->dfs_root);
5229
5230	if (tc->netdevice_nb.notifier_call)
5231	unregister_netdevice_notifier_dev_net(dev: priv->netdev,
5232	nb: &tc->netdevice_nb,
5233	nn: &tc->netdevice_nn);
5234
5235	mlx5e_mod_hdr_tbl_destroy(tbl: &tc->mod_hdr);
5236	mutex_destroy(lock: &tc->hairpin_tbl_lock);
5237
5238	rhashtable_free_and_destroy(ht: &tc->ht, free_fn: _mlx5e_tc_del_flow, NULL);
5239
5240	if (!IS_ERR_OR_NULL(ptr: tc->t)) {
5241	mlx5_chains_put_table(chains: tc->chains, chain: 0, prio: 1, level: MLX5E_TC_FT_LEVEL);
5242	tc->t = NULL;
5243	}
5244	mutex_destroy(lock: &tc->t_lock);
5245
5246	mlx5_tc_ct_clean(ct_priv: tc->ct);
5247	mlx5e_tc_post_act_destroy(post_act: tc->post_act);
5248	mapping_destroy(ctx: tc->mapping);
5249	mlx5_chains_destroy(chains: tc->chains);
5250	mlx5e_tc_nic_destroy_miss_table(priv);
5251	mlx5e_tc_act_stats_free(handle: tc->action_stats_handle);
5252	}
5253
5254	int mlx5e_tc_ht_init(struct rhashtable *tc_ht)
5255	{
5256	int err;
5257
5258	err = rhashtable_init(ht: tc_ht, params: &tc_ht_params);
5259	if (err)
5260	return err;
5261
5262	lockdep_set_class(&tc_ht->mutex, &tc_ht_lock_key);
5263	lockdep_init_map(lock: &tc_ht->run_work.lockdep_map, name: "tc_ht_wq_key", key: &tc_ht_wq_key, subclass: 0);
5264
5265	return 0;
5266	}
5267
5268	void mlx5e_tc_ht_cleanup(struct rhashtable *tc_ht)
5269	{
5270	rhashtable_free_and_destroy(ht: tc_ht, free_fn: _mlx5e_tc_del_flow, NULL);
5271	}
5272
5273	int mlx5e_tc_esw_init(struct mlx5_rep_uplink_priv *uplink_priv)
5274	{
5275	const size_t sz_enc_opts = sizeof(struct tunnel_match_enc_opts);
5276	struct netdev_phys_item_id ppid;
5277	struct mlx5e_rep_priv *rpriv;
5278	struct mapping_ctx *mapping;
5279	struct mlx5_eswitch *esw;
5280	struct mlx5e_priv *priv;
5281	u64 mapping_id, key;
5282	int err = 0;
5283
5284	rpriv = container_of(uplink_priv, struct mlx5e_rep_priv, uplink_priv);
5285	priv = netdev_priv(dev: rpriv->netdev);
5286	esw = priv->mdev->priv.eswitch;
5287
5288	uplink_priv->post_act = mlx5e_tc_post_act_init(priv, esw_chains(esw),
5289	ns_type: MLX5_FLOW_NAMESPACE_FDB);
5290	uplink_priv->ct_priv = mlx5_tc_ct_init(priv: netdev_priv(dev: priv->netdev),
5291	esw_chains(esw),
5292	mod_hdr: &esw->offloads.mod_hdr,
5293	ns_type: MLX5_FLOW_NAMESPACE_FDB,
5294	post_act: uplink_priv->post_act);
5295
5296	uplink_priv->int_port_priv = mlx5e_tc_int_port_init(priv: netdev_priv(dev: priv->netdev));
5297
5298	uplink_priv->tc_psample = mlx5e_tc_sample_init(esw, post_act: uplink_priv->post_act);
5299
5300	mapping_id = mlx5_query_nic_system_image_guid(mdev: esw->dev);
5301
5302	mapping = mapping_create_for_id(id: mapping_id, type: MAPPING_TYPE_TUNNEL,
5303	data_size: sizeof(struct tunnel_match_key),
5304	TUNNEL_INFO_BITS_MASK, delayed_removal: true);
5305
5306	if (IS_ERR(ptr: mapping)) {
5307	err = PTR_ERR(ptr: mapping);
5308	goto err_tun_mapping;
5309	}
5310	uplink_priv->tunnel_mapping = mapping;
5311
5312	/* Two last values are reserved for stack devices slow path table mark
5313	* and bridge ingress push mark.
5314	*/
5315	mapping = mapping_create_for_id(id: mapping_id, type: MAPPING_TYPE_TUNNEL_ENC_OPTS,
5316	data_size: sz_enc_opts, ENC_OPTS_BITS_MASK - 2, delayed_removal: true);
5317	if (IS_ERR(ptr: mapping)) {
5318	err = PTR_ERR(ptr: mapping);
5319	goto err_enc_opts_mapping;
5320	}
5321	uplink_priv->tunnel_enc_opts_mapping = mapping;
5322
5323	uplink_priv->encap = mlx5e_tc_tun_init(priv);
5324	if (IS_ERR(ptr: uplink_priv->encap)) {
5325	err = PTR_ERR(ptr: uplink_priv->encap);
5326	goto err_register_fib_notifier;
5327	}
5328
5329	uplink_priv->action_stats_handle = mlx5e_tc_act_stats_create();
5330	if (IS_ERR(ptr: uplink_priv->action_stats_handle)) {
5331	err = PTR_ERR(ptr: uplink_priv->action_stats_handle);
5332	goto err_action_counter;
5333	}
5334
5335	err = dev_get_port_parent_id(dev: priv->netdev, ppid: &ppid, recurse: false);
5336	if (!err) {
5337	memcpy(&key, &ppid.id, sizeof(key));
5338	mlx5_esw_offloads_devcom_init(esw, key);
5339	}
5340
5341	return 0;
5342
5343	err_action_counter:
5344	mlx5e_tc_tun_cleanup(encap: uplink_priv->encap);
5345	err_register_fib_notifier:
5346	mapping_destroy(ctx: uplink_priv->tunnel_enc_opts_mapping);
5347	err_enc_opts_mapping:
5348	mapping_destroy(ctx: uplink_priv->tunnel_mapping);
5349	err_tun_mapping:
5350	mlx5e_tc_sample_cleanup(tc_psample: uplink_priv->tc_psample);
5351	mlx5e_tc_int_port_cleanup(priv: uplink_priv->int_port_priv);
5352	mlx5_tc_ct_clean(ct_priv: uplink_priv->ct_priv);
5353	netdev_warn(dev: priv->netdev,
5354	format: "Failed to initialize tc (eswitch), err: %d", err);
5355	mlx5e_tc_post_act_destroy(post_act: uplink_priv->post_act);
5356	return err;
5357	}
5358
5359	void mlx5e_tc_esw_cleanup(struct mlx5_rep_uplink_priv *uplink_priv)
5360	{
5361	struct mlx5e_rep_priv *rpriv;
5362	struct mlx5_eswitch *esw;
5363	struct mlx5e_priv *priv;
5364
5365	rpriv = container_of(uplink_priv, struct mlx5e_rep_priv, uplink_priv);
5366	priv = netdev_priv(dev: rpriv->netdev);
5367	esw = priv->mdev->priv.eswitch;
5368
5369	mlx5_esw_offloads_devcom_cleanup(esw);
5370
5371	mlx5e_tc_tun_cleanup(encap: uplink_priv->encap);
5372
5373	mapping_destroy(ctx: uplink_priv->tunnel_enc_opts_mapping);
5374	mapping_destroy(ctx: uplink_priv->tunnel_mapping);
5375
5376	mlx5e_tc_sample_cleanup(tc_psample: uplink_priv->tc_psample);
5377	mlx5e_tc_int_port_cleanup(priv: uplink_priv->int_port_priv);
5378	mlx5_tc_ct_clean(ct_priv: uplink_priv->ct_priv);
5379	mlx5e_flow_meters_cleanup(flow_meters: uplink_priv->flow_meters);
5380	mlx5e_tc_post_act_destroy(post_act: uplink_priv->post_act);
5381	mlx5e_tc_act_stats_free(handle: uplink_priv->action_stats_handle);
5382	}
5383
5384	int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags)
5385	{
5386	struct rhashtable *tc_ht = get_tc_ht(priv, flags);
5387
5388	return atomic_read(v: &tc_ht->nelems);
5389	}
5390
5391	void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw)
5392	{
5393	struct mlx5e_tc_flow *flow, *tmp;
5394	int i;
5395
5396	for (i = 0; i < MLX5_MAX_PORTS; i++) {
5397	if (i == mlx5_get_dev_index(dev: esw->dev))
5398	continue;
5399	list_for_each_entry_safe(flow, tmp, &esw->offloads.peer_flows[i], peer[i])
5400	mlx5e_tc_del_fdb_peers_flow(flow);
5401	}
5402	}
5403
5404	void mlx5e_tc_reoffload_flows_work(struct work_struct *work)
5405	{
5406	struct mlx5_rep_uplink_priv *rpriv =
5407	container_of(work, struct mlx5_rep_uplink_priv,
5408	reoffload_flows_work);
5409	struct mlx5e_tc_flow *flow, *tmp;
5410
5411	mutex_lock(&rpriv->unready_flows_lock);
5412	list_for_each_entry_safe(flow, tmp, &rpriv->unready_flows, unready) {
5413	if (!mlx5e_tc_add_fdb_flow(priv: flow->priv, flow, NULL))
5414	unready_flow_del(flow);
5415	}
5416	mutex_unlock(lock: &rpriv->unready_flows_lock);
5417	}
5418
5419	static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv,
5420	struct flow_cls_offload *cls_flower,
5421	unsigned long flags)
5422	{
5423	switch (cls_flower->command) {
5424	case FLOW_CLS_REPLACE:
5425	return mlx5e_configure_flower(dev: priv->netdev, priv, f: cls_flower,
5426	flags);
5427	case FLOW_CLS_DESTROY:
5428	return mlx5e_delete_flower(dev: priv->netdev, priv, f: cls_flower,
5429	flags);
5430	case FLOW_CLS_STATS:
5431	return mlx5e_stats_flower(dev: priv->netdev, priv, f: cls_flower,
5432	flags);
5433	default:
5434	return -EOPNOTSUPP;
5435	}
5436	}
5437
5438	int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
5439	void *cb_priv)
5440	{
5441	unsigned long flags = MLX5_TC_FLAG(INGRESS);
5442	struct mlx5e_priv *priv = cb_priv;
5443
5444	if (!priv->netdev || !netif_device_present(dev: priv->netdev))
5445	return -EOPNOTSUPP;
5446
5447	if (mlx5e_is_uplink_rep(priv))
5448	flags |= MLX5_TC_FLAG(ESW_OFFLOAD);
5449	else
5450	flags |= MLX5_TC_FLAG(NIC_OFFLOAD);
5451
5452	switch (type) {
5453	case TC_SETUP_CLSFLOWER:
5454	return mlx5e_setup_tc_cls_flower(priv, cls_flower: type_data, flags);
5455	default:
5456	return -EOPNOTSUPP;
5457	}
5458	}
5459
5460	static bool mlx5e_tc_restore_tunnel(struct mlx5e_priv *priv, struct sk_buff *skb,
5461	struct mlx5e_tc_update_priv *tc_priv,
5462	u32 tunnel_id)
5463	{
5464	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
5465	struct tunnel_match_enc_opts enc_opts = {};
5466	struct mlx5_rep_uplink_priv *uplink_priv;
5467	struct mlx5e_rep_priv *uplink_rpriv;
5468	struct metadata_dst *tun_dst;
5469	struct tunnel_match_key key;
5470	u32 tun_id, enc_opts_id;
5471	struct net_device *dev;
5472	int err;
5473
5474	enc_opts_id = tunnel_id & ENC_OPTS_BITS_MASK;
5475	tun_id = tunnel_id >> ENC_OPTS_BITS;
5476
5477	if (!tun_id)
5478	return true;
5479
5480	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
5481	uplink_priv = &uplink_rpriv->uplink_priv;
5482
5483	err = mapping_find(ctx: uplink_priv->tunnel_mapping, id: tun_id, data: &key);
5484	if (err) {
5485	netdev_dbg(priv->netdev,
5486	"Couldn't find tunnel for tun_id: %d, err: %d\n",
5487	tun_id, err);
5488	return false;
5489	}
5490
5491	if (enc_opts_id) {
5492	err = mapping_find(ctx: uplink_priv->tunnel_enc_opts_mapping,
5493	id: enc_opts_id, data: &enc_opts);
5494	if (err) {
5495	netdev_dbg(priv->netdev,
5496	"Couldn't find tunnel (opts) for tun_id: %d, err: %d\n",
5497	enc_opts_id, err);
5498	return false;
5499	}
5500	}
5501
5502	switch (key.enc_control.addr_type) {
5503	case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
5504	tun_dst = __ip_tun_set_dst(saddr: key.enc_ipv4.src, daddr: key.enc_ipv4.dst,
5505	tos: key.enc_ip.tos, ttl: key.enc_ip.ttl,
5506	tp_dst: key.enc_tp.dst, TUNNEL_KEY,
5507	tunnel_id: key32_to_tunnel_id(key: key.enc_key_id.keyid),
5508	md_size: enc_opts.key.len);
5509	break;
5510	case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
5511	tun_dst = __ipv6_tun_set_dst(saddr: &key.enc_ipv6.src, daddr: &key.enc_ipv6.dst,
5512	tos: key.enc_ip.tos, ttl: key.enc_ip.ttl,
5513	tp_dst: key.enc_tp.dst, label: 0, TUNNEL_KEY,
5514	tunnel_id: key32_to_tunnel_id(key: key.enc_key_id.keyid),
5515	md_size: enc_opts.key.len);
5516	break;
5517	default:
5518	netdev_dbg(priv->netdev,
5519	"Couldn't restore tunnel, unsupported addr_type: %d\n",
5520	key.enc_control.addr_type);
5521	return false;
5522	}
5523
5524	if (!tun_dst) {
5525	netdev_dbg(priv->netdev, "Couldn't restore tunnel, no tun_dst\n");
5526	return false;
5527	}
5528
5529	tun_dst->u.tun_info.key.tp_src = key.enc_tp.src;
5530
5531	if (enc_opts.key.len)
5532	ip_tunnel_info_opts_set(info: &tun_dst->u.tun_info,
5533	from: enc_opts.key.data,
5534	len: enc_opts.key.len,
5535	flags: enc_opts.key.dst_opt_type);
5536
5537	skb_dst_set(skb, dst: (struct dst_entry *)tun_dst);
5538	dev = dev_get_by_index(net: &init_net, ifindex: key.filter_ifindex);
5539	if (!dev) {
5540	netdev_dbg(priv->netdev,
5541	"Couldn't find tunnel device with ifindex: %d\n",
5542	key.filter_ifindex);
5543	return false;
5544	}
5545
5546	/* Set fwd_dev so we do dev_put() after datapath */
5547	tc_priv->fwd_dev = dev;
5548
5549	skb->dev = dev;
5550
5551	return true;
5552	}
5553
5554	static bool mlx5e_tc_restore_skb_tc_meta(struct sk_buff *skb, struct mlx5_tc_ct_priv *ct_priv,
5555	struct mlx5_mapped_obj *mapped_obj, u32 zone_restore_id,
5556	u32 tunnel_id, struct mlx5e_tc_update_priv *tc_priv)
5557	{
5558	struct mlx5e_priv *priv = netdev_priv(dev: skb->dev);
5559	struct tc_skb_ext *tc_skb_ext;
5560	u64 act_miss_cookie;
5561	u32 chain;
5562
5563	chain = mapped_obj->type == MLX5_MAPPED_OBJ_CHAIN ? mapped_obj->chain : 0;
5564	act_miss_cookie = mapped_obj->type == MLX5_MAPPED_OBJ_ACT_MISS ?
5565	mapped_obj->act_miss_cookie : 0;
5566	if (chain || act_miss_cookie) {
5567	if (!mlx5e_tc_ct_restore_flow(ct_priv, skb, zone_restore_id))
5568	return false;
5569
5570	tc_skb_ext = tc_skb_ext_alloc(skb);
5571	if (!tc_skb_ext) {
5572	WARN_ON(1);
5573	return false;
5574	}
5575
5576	if (act_miss_cookie) {
5577	tc_skb_ext->act_miss_cookie = act_miss_cookie;
5578	tc_skb_ext->act_miss = 1;
5579	} else {
5580	tc_skb_ext->chain = chain;
5581	}
5582	}
5583
5584	if (tc_priv)
5585	return mlx5e_tc_restore_tunnel(priv, skb, tc_priv, tunnel_id);
5586
5587	return true;
5588	}
5589
5590	static void mlx5e_tc_restore_skb_sample(struct mlx5e_priv *priv, struct sk_buff *skb,
5591	struct mlx5_mapped_obj *mapped_obj,
5592	struct mlx5e_tc_update_priv *tc_priv)
5593	{
5594	if (!mlx5e_tc_restore_tunnel(priv, skb, tc_priv, tunnel_id: mapped_obj->sample.tunnel_id)) {
5595	netdev_dbg(priv->netdev,
5596	"Failed to restore tunnel info for sampled packet\n");
5597	return;
5598	}
5599	mlx5e_tc_sample_skb(skb, mapped_obj);
5600	}
5601
5602	static bool mlx5e_tc_restore_skb_int_port(struct mlx5e_priv *priv, struct sk_buff *skb,
5603	struct mlx5_mapped_obj *mapped_obj,
5604	struct mlx5e_tc_update_priv *tc_priv,
5605	u32 tunnel_id)
5606	{
5607	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
5608	struct mlx5_rep_uplink_priv *uplink_priv;
5609	struct mlx5e_rep_priv *uplink_rpriv;
5610	bool forward_tx = false;
5611
5612	/* Tunnel restore takes precedence over int port restore */
5613	if (tunnel_id)
5614	return mlx5e_tc_restore_tunnel(priv, skb, tc_priv, tunnel_id);
5615
5616	uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, rep_type: REP_ETH);
5617	uplink_priv = &uplink_rpriv->uplink_priv;
5618
5619	if (mlx5e_tc_int_port_dev_fwd(priv: uplink_priv->int_port_priv, skb,
5620	int_vport_metadata: mapped_obj->int_port_metadata, forward_tx: &forward_tx)) {
5621	/* Set fwd_dev for future dev_put */
5622	tc_priv->fwd_dev = skb->dev;
5623	tc_priv->forward_tx = forward_tx;
5624
5625	return true;
5626	}
5627
5628	return false;
5629	}
5630
5631	bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, struct sk_buff *skb,
5632	struct mapping_ctx *mapping_ctx, u32 mapped_obj_id,
5633	struct mlx5_tc_ct_priv *ct_priv,
5634	u32 zone_restore_id, u32 tunnel_id,
5635	struct mlx5e_tc_update_priv *tc_priv)
5636	{
5637	struct mlx5e_priv *priv = netdev_priv(dev: skb->dev);
5638	struct mlx5_mapped_obj mapped_obj;
5639	int err;
5640
5641	err = mapping_find(ctx: mapping_ctx, id: mapped_obj_id, data: &mapped_obj);
5642	if (err) {
5643	netdev_dbg(skb->dev,
5644	"Couldn't find mapped object for mapped_obj_id: %d, err: %d\n",
5645	mapped_obj_id, err);
5646	return false;
5647	}
5648
5649	switch (mapped_obj.type) {
5650	case MLX5_MAPPED_OBJ_CHAIN:
5651	case MLX5_MAPPED_OBJ_ACT_MISS:
5652	return mlx5e_tc_restore_skb_tc_meta(skb, ct_priv, mapped_obj: &mapped_obj, zone_restore_id,
5653	tunnel_id, tc_priv);
5654	case MLX5_MAPPED_OBJ_SAMPLE:
5655	mlx5e_tc_restore_skb_sample(priv, skb, mapped_obj: &mapped_obj, tc_priv);
5656	tc_priv->skb_done = true;
5657	return true;
5658	case MLX5_MAPPED_OBJ_INT_PORT_METADATA:
5659	return mlx5e_tc_restore_skb_int_port(priv, skb, mapped_obj: &mapped_obj, tc_priv, tunnel_id);
5660	default:
5661	netdev_dbg(priv->netdev, "Invalid mapped object type: %d\n", mapped_obj.type);
5662	return false;
5663	}
5664
5665	return false;
5666	}
5667
5668	bool mlx5e_tc_update_skb_nic(struct mlx5_cqe64 *cqe, struct sk_buff *skb)
5669	{
5670	struct mlx5e_priv *priv = netdev_priv(dev: skb->dev);
5671	u32 mapped_obj_id, reg_b, zone_restore_id;
5672	struct mlx5_tc_ct_priv *ct_priv;
5673	struct mapping_ctx *mapping_ctx;
5674	struct mlx5e_tc_table *tc;
5675
5676	reg_b = be32_to_cpu(cqe->ft_metadata);
5677	tc = mlx5e_fs_get_tc(fs: priv->fs);
5678	mapped_obj_id = reg_b & MLX5E_TC_TABLE_CHAIN_TAG_MASK;
5679	zone_restore_id = (reg_b >> MLX5_REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) &
5680	ESW_ZONE_ID_MASK;
5681	ct_priv = tc->ct;
5682	mapping_ctx = tc->mapping;
5683
5684	return mlx5e_tc_update_skb(cqe, skb, mapping_ctx, mapped_obj_id, ct_priv, zone_restore_id,
5685	tunnel_id: 0, NULL);
5686	}
5687
5688	static struct mapping_ctx *
5689	mlx5e_get_priv_obj_mapping(struct mlx5e_priv *priv)
5690	{
5691	struct mlx5e_tc_table *tc;
5692	struct mlx5_eswitch *esw;
5693	struct mapping_ctx *ctx;
5694
5695	if (is_mdev_switchdev_mode(dev: priv->mdev)) {
5696	esw = priv->mdev->priv.eswitch;
5697	ctx = esw->offloads.reg_c0_obj_pool;
5698	} else {
5699	tc = mlx5e_fs_get_tc(fs: priv->fs);
5700	ctx = tc->mapping;
5701	}
5702
5703	return ctx;
5704	}
5705
5706	int mlx5e_tc_action_miss_mapping_get(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr,
5707	u64 act_miss_cookie, u32 *act_miss_mapping)
5708	{
5709	struct mlx5_mapped_obj mapped_obj = {};
5710	struct mlx5_eswitch *esw;
5711	struct mapping_ctx *ctx;
5712	int err;
5713
5714	ctx = mlx5e_get_priv_obj_mapping(priv);
5715	mapped_obj.type = MLX5_MAPPED_OBJ_ACT_MISS;
5716	mapped_obj.act_miss_cookie = act_miss_cookie;
5717	err = mapping_add(ctx, data: &mapped_obj, id: act_miss_mapping);
5718	if (err)
5719	return err;
5720
5721	if (!is_mdev_switchdev_mode(dev: priv->mdev))
5722	return 0;
5723
5724	esw = priv->mdev->priv.eswitch;
5725	attr->act_id_restore_rule = esw_add_restore_rule(esw, tag: *act_miss_mapping);
5726	if (IS_ERR(ptr: attr->act_id_restore_rule)) {
5727	err = PTR_ERR(ptr: attr->act_id_restore_rule);
5728	goto err_rule;
5729	}
5730
5731	return 0;
5732
5733	err_rule:
5734	mapping_remove(ctx, id: *act_miss_mapping);
5735	return err;
5736	}
5737
5738	void mlx5e_tc_action_miss_mapping_put(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr,
5739	u32 act_miss_mapping)
5740	{
5741	struct mapping_ctx *ctx = mlx5e_get_priv_obj_mapping(priv);
5742
5743	if (is_mdev_switchdev_mode(dev: priv->mdev))
5744	mlx5_del_flow_rules(fr: attr->act_id_restore_rule);
5745	mapping_remove(ctx, id: act_miss_mapping);
5746	}
5747