1	// SPDX-License-Identifier: GPL-2.0-only
2	/****************************************************************************
3	* Driver for Solarflare network controllers and boards
4	* Copyright 2019 Solarflare Communications Inc.
5	* Copyright 2020-2022 Xilinx Inc.
6	*
7	* This program is free software; you can redistribute it and/or modify it
8	* under the terms of the GNU General Public License version 2 as published
9	* by the Free Software Foundation, incorporated herein by reference.
10	*/
11
12	#include <net/pkt_cls.h>
13	#include <net/vxlan.h>
14	#include <net/geneve.h>
15	#include <net/tc_act/tc_ct.h>
16	#include "tc.h"
17	#include "tc_bindings.h"
18	#include "tc_encap_actions.h"
19	#include "tc_conntrack.h"
20	#include "mae.h"
21	#include "ef100_rep.h"
22	#include "efx.h"
23
24	enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)
25	{
26	if (netif_is_vxlan(dev: net_dev))
27	return EFX_ENCAP_TYPE_VXLAN;
28	if (netif_is_geneve(dev: net_dev))
29	return EFX_ENCAP_TYPE_GENEVE;
30
31	return EFX_ENCAP_TYPE_NONE;
32	}
33
34	#define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff)
35	/* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */
36	#define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000)
37	#define EFX_EFV_PF NULL
38	/* Look up the representor information (efv) for a device.
39	* May return NULL for the PF (us), or an error pointer for a device that
40	* isn't supported as a TC offload endpoint
41	*/
42	struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
43	struct net_device *dev)
44	{
45	struct efx_rep *efv;
46
47	if (!dev)
48	return ERR_PTR(error: -EOPNOTSUPP);
49	/* Is it us (the PF)? */
50	if (dev == efx->net_dev)
51	return EFX_EFV_PF;
52	/* Is it an efx vfrep at all? */
53	if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)
54	return ERR_PTR(error: -EOPNOTSUPP);
55	/* Is it ours? We don't support TC rules that include another
56	* EF100's netdevices (not even on another port of the same NIC).
57	*/
58	efv = netdev_priv(dev);
59	if (efv->parent != efx)
60	return ERR_PTR(error: -EOPNOTSUPP);
61	return efv;
62	}
63
64	/* Convert a driver-internal vport ID into an internal device (PF or VF) */
65	static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)
66	{
67	u32 mport;
68
69	if (IS_ERR(ptr: efv))
70	return PTR_ERR(ptr: efv);
71	if (!efv) /* device is PF (us) */
72	efx_mae_mport_uplink(efx, out: &mport);
73	else /* device is repr */
74	efx_mae_mport_mport(efx, mport_id: efv->mport, out: &mport);
75	return mport;
76	}
77
78	/* Convert a driver-internal vport ID into an external device (wire or VF) */
79	s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
80	{
81	u32 mport;
82
83	if (IS_ERR(ptr: efv))
84	return PTR_ERR(ptr: efv);
85	if (!efv) /* device is PF (us) */
86	efx_mae_mport_wire(efx, out: &mport);
87	else /* device is repr */
88	efx_mae_mport_mport(efx, mport_id: efv->mport, out: &mport);
89	return mport;
90	}
91
92	static const struct rhashtable_params efx_tc_mac_ht_params = {
93	.key_len = offsetofend(struct efx_tc_mac_pedit_action, h_addr),
94	.key_offset = 0,
95	.head_offset = offsetof(struct efx_tc_mac_pedit_action, linkage),
96	};
97
98	static const struct rhashtable_params efx_tc_encap_match_ht_params = {
99	.key_len = offsetof(struct efx_tc_encap_match, linkage),
100	.key_offset = 0,
101	.head_offset = offsetof(struct efx_tc_encap_match, linkage),
102	};
103
104	static const struct rhashtable_params efx_tc_match_action_ht_params = {
105	.key_len = sizeof(unsigned long),
106	.key_offset = offsetof(struct efx_tc_flow_rule, cookie),
107	.head_offset = offsetof(struct efx_tc_flow_rule, linkage),
108	};
109
110	static const struct rhashtable_params efx_tc_lhs_rule_ht_params = {
111	.key_len = sizeof(unsigned long),
112	.key_offset = offsetof(struct efx_tc_lhs_rule, cookie),
113	.head_offset = offsetof(struct efx_tc_lhs_rule, linkage),
114	};
115
116	static const struct rhashtable_params efx_tc_recirc_ht_params = {
117	.key_len = offsetof(struct efx_tc_recirc_id, linkage),
118	.key_offset = 0,
119	.head_offset = offsetof(struct efx_tc_recirc_id, linkage),
120	};
121
122	static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx,
123	unsigned char h_addr[ETH_ALEN],
124	struct netlink_ext_ack *extack)
125	{
126	struct efx_tc_mac_pedit_action *ped, *old;
127	int rc;
128
129	ped = kzalloc(size: sizeof(*ped), GFP_USER);
130	if (!ped)
131	return ERR_PTR(error: -ENOMEM);
132	memcpy(ped->h_addr, h_addr, ETH_ALEN);
133	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->mac_ht,
134	obj: &ped->linkage,
135	params: efx_tc_mac_ht_params);
136	if (old) {
137	/* don't need our new entry */
138	kfree(objp: ped);
139	if (IS_ERR(ptr: old)) /* oh dear, it's actually an error */
140	return ERR_CAST(ptr: old);
141	if (!refcount_inc_not_zero(r: &old->ref))
142	return ERR_PTR(error: -EAGAIN);
143	/* existing entry found, ref taken */
144	return old;
145	}
146
147	rc = efx_mae_allocate_pedit_mac(efx, ped);
148	if (rc < 0) {
149	NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw");
150	goto out_remove;
151	}
152
153	/* ref and return */
154	refcount_set(r: &ped->ref, n: 1);
155	return ped;
156	out_remove:
157	rhashtable_remove_fast(ht: &efx->tc->mac_ht, obj: &ped->linkage,
158	params: efx_tc_mac_ht_params);
159	kfree(objp: ped);
160	return ERR_PTR(error: rc);
161	}
162
163	static void efx_tc_flower_put_mac(struct efx_nic *efx,
164	struct efx_tc_mac_pedit_action *ped)
165	{
166	if (!refcount_dec_and_test(r: &ped->ref))
167	return; /* still in use */
168	rhashtable_remove_fast(ht: &efx->tc->mac_ht, obj: &ped->linkage,
169	params: efx_tc_mac_ht_params);
170	efx_mae_free_pedit_mac(efx, ped);
171	kfree(objp: ped);
172	}
173
174	static void efx_tc_free_action_set(struct efx_nic *efx,
175	struct efx_tc_action_set *act, bool in_hw)
176	{
177	/* Failure paths calling this on the 'cursor' action set in_hw=false,
178	* because if the alloc had succeeded we'd've put it in acts.list and
179	* not still have it in act.
180	*/
181	if (in_hw) {
182	efx_mae_free_action_set(efx, fw_id: act->fw_id);
183	/* in_hw is true iff we are on an acts.list; make sure to
184	* remove ourselves from that list before we are freed.
185	*/
186	list_del(entry: &act->list);
187	}
188	if (act->count) {
189	spin_lock_bh(lock: &act->count->cnt->lock);
190	if (!list_empty(head: &act->count_user))
191	list_del(entry: &act->count_user);
192	spin_unlock_bh(lock: &act->count->cnt->lock);
193	efx_tc_flower_put_counter_index(efx, ctr: act->count);
194	}
195	if (act->encap_md) {
196	list_del(entry: &act->encap_user);
197	efx_tc_flower_release_encap_md(efx, encap: act->encap_md);
198	}
199	if (act->src_mac)
200	efx_tc_flower_put_mac(efx, ped: act->src_mac);
201	if (act->dst_mac)
202	efx_tc_flower_put_mac(efx, ped: act->dst_mac);
203	kfree(objp: act);
204	}
205
206	static void efx_tc_free_action_set_list(struct efx_nic *efx,
207	struct efx_tc_action_set_list *acts,
208	bool in_hw)
209	{
210	struct efx_tc_action_set *act, *next;
211
212	/* Failure paths set in_hw=false, because usually the acts didn't get
213	* to efx_mae_alloc_action_set_list(); if they did, the failure tree
214	* has a separate efx_mae_free_action_set_list() before calling us.
215	*/
216	if (in_hw)
217	efx_mae_free_action_set_list(efx, acts);
218	/* Any act that's on the list will be in_hw even if the list isn't */
219	list_for_each_entry_safe(act, next, &acts->list, list)
220	efx_tc_free_action_set(efx, act, in_hw: true);
221	/* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */
222	}
223
224	/* Boilerplate for the simple 'copy a field' cases */
225	#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
226	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) { \
227	struct flow_match_##_type fm; \
228	\
229	flow_rule_match_##_tcget(rule, &fm); \
230	match->value._field = fm.key->_tcfield; \
231	match->mask._field = fm.mask->_tcfield; \
232	}
233	#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field) \
234	_MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)
235	#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field) \
236	_MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)
237
238	static int efx_tc_flower_parse_match(struct efx_nic *efx,
239	struct flow_rule *rule,
240	struct efx_tc_match *match,
241	struct netlink_ext_ack *extack)
242	{
243	struct flow_dissector *dissector = rule->match.dissector;
244	unsigned char ipv = 0;
245
246	/* Owing to internal TC infelicities, the IPV6_ADDRS key might be set
247	* even on IPv4 filters; so rather than relying on dissector->used_keys
248	* we check the addr_type in the CONTROL key. If we don't find it (or
249	* it's masked, which should never happen), we treat both IPV4_ADDRS
250	* and IPV6_ADDRS as absent.
251	*/
252	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CONTROL)) {
253	struct flow_match_control fm;
254
255	flow_rule_match_control(rule, out: &fm);
256	if (IS_ALL_ONES(fm.mask->addr_type))
257	switch (fm.key->addr_type) {
258	case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
259	ipv = 4;
260	break;
261	case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
262	ipv = 6;
263	break;
264	default:
265	break;
266	}
267
268	if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {
269	match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;
270	match->mask.ip_frag = true;
271	}
272	if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {
273	match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;
274	match->mask.ip_firstfrag = true;
275	}
276	if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) {
277	NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x",
278	fm.mask->flags);
279	return -EOPNOTSUPP;
280	}
281	}
282	if (dissector->used_keys &
283	~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
284	BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
285	BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
286	BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
287	BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) |
288	BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
289	BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
290	BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
291	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
292	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
293	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
294	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
295	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) |
296	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
297	BIT_ULL(FLOW_DISSECTOR_KEY_CT) |
298	BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
299	BIT_ULL(FLOW_DISSECTOR_KEY_IP))) {
300	NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx",
301	dissector->used_keys);
302	return -EOPNOTSUPP;
303	}
304
305	MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);
306	/* Make sure we're IP if any L3/L4 keys used. */
307	if (!IS_ALL_ONES(match->mask.eth_proto) ||
308	!(match->value.eth_proto == htons(ETH_P_IP) ||
309	match->value.eth_proto == htons(ETH_P_IPV6)))
310	if (dissector->used_keys &
311	(BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
312	BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
313	BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
314	BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
315	BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
316	NL_SET_ERR_MSG_FMT_MOD(extack,
317	"L3/L4 flower keys %#llx require protocol ipv[46]",
318	dissector->used_keys);
319	return -EINVAL;
320	}
321
322	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_VLAN)) {
323	struct flow_match_vlan fm;
324
325	flow_rule_match_vlan(rule, out: &fm);
326	if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
327	match->value.vlan_proto[0] = fm.key->vlan_tpid;
328	match->mask.vlan_proto[0] = fm.mask->vlan_tpid;
329	match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |
330	fm.key->vlan_id);
331	match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |
332	fm.mask->vlan_id);
333	}
334	}
335
336	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CVLAN)) {
337	struct flow_match_vlan fm;
338
339	flow_rule_match_cvlan(rule, out: &fm);
340	if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
341	match->value.vlan_proto[1] = fm.key->vlan_tpid;
342	match->mask.vlan_proto[1] = fm.mask->vlan_tpid;
343	match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |
344	fm.key->vlan_id);
345	match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |
346	fm.mask->vlan_id);
347	}
348	}
349
350	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
351	struct flow_match_eth_addrs fm;
352
353	flow_rule_match_eth_addrs(rule, out: &fm);
354	ether_addr_copy(dst: match->value.eth_saddr, src: fm.key->src);
355	ether_addr_copy(dst: match->value.eth_daddr, src: fm.key->dst);
356	ether_addr_copy(dst: match->mask.eth_saddr, src: fm.mask->src);
357	ether_addr_copy(dst: match->mask.eth_daddr, src: fm.mask->dst);
358	}
359
360	MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);
361	/* Make sure we're TCP/UDP if any L4 keys used. */
362	if ((match->value.ip_proto != IPPROTO_UDP &&
363	match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))
364	if (dissector->used_keys &
365	(BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
366	BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
367	NL_SET_ERR_MSG_FMT_MOD(extack,
368	"L4 flower keys %#llx require ipproto udp or tcp",
369	dissector->used_keys);
370	return -EINVAL;
371	}
372	MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);
373	MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);
374	if (ipv == 4) {
375	MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);
376	MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);
377	}
378	#ifdef CONFIG_IPV6
379	else if (ipv == 6) {
380	MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);
381	MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);
382	}
383	#endif
384	MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
385	MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
386	MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
387	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
388	struct flow_match_control fm;
389
390	flow_rule_match_enc_control(rule, out: &fm);
391	if (fm.mask->flags) {
392	NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on enc_control.flags %#x",
393	fm.mask->flags);
394	return -EOPNOTSUPP;
395	}
396	if (!IS_ALL_ONES(fm.mask->addr_type)) {
397	NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",
398	fm.mask->addr_type,
399	fm.key->addr_type);
400	return -EOPNOTSUPP;
401	}
402	switch (fm.key->addr_type) {
403	case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
404	MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
405	src, enc_src_ip);
406	MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
407	dst, enc_dst_ip);
408	break;
409	#ifdef CONFIG_IPV6
410	case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
411	MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
412	src, enc_src_ip6);
413	MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
414	dst, enc_dst_ip6);
415	break;
416	#endif
417	default:
418	NL_SET_ERR_MSG_FMT_MOD(extack,
419	"Unsupported enc addr_type %u (supported are IPv4, IPv6)",
420	fm.key->addr_type);
421	return -EOPNOTSUPP;
422	}
423	MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);
424	MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);
425	MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);
426	MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);
427	MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);
428	} else if (dissector->used_keys &
429	(BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
430	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
431	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
432	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
433	BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) {
434	NL_SET_ERR_MSG_FMT_MOD(extack,
435	"Flower enc keys require enc_control (keys: %#llx)",
436	dissector->used_keys);
437	return -EOPNOTSUPP;
438	}
439	if (flow_rule_match_key(rule, key: FLOW_DISSECTOR_KEY_CT)) {
440	struct flow_match_ct fm;
441
442	flow_rule_match_ct(rule, out: &fm);
443	match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
444	match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
445	match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
446	match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
447	if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
448	TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) {
449	NL_SET_ERR_MSG_FMT_MOD(extack,
450	"Unsupported ct_state match %#x",
451	fm.mask->ct_state);
452	return -EOPNOTSUPP;
453	}
454	match->value.ct_mark = fm.key->ct_mark;
455	match->mask.ct_mark = fm.mask->ct_mark;
456	match->value.ct_zone = fm.key->ct_zone;
457	match->mask.ct_zone = fm.mask->ct_zone;
458
459	if (memchr_inv(p: fm.mask->ct_labels, c: 0, size: sizeof(fm.mask->ct_labels))) {
460	NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported");
461	return -EOPNOTSUPP;
462	}
463	}
464
465	return 0;
466	}
467
468	static void efx_tc_flower_release_encap_match(struct efx_nic *efx,
469	struct efx_tc_encap_match *encap)
470	{
471	int rc;
472
473	if (!refcount_dec_and_test(r: &encap->ref))
474	return; /* still in use */
475
476	if (encap->type == EFX_TC_EM_DIRECT) {
477	rc = efx_mae_unregister_encap_match(efx, encap);
478	if (rc)
479	/* Display message but carry on and remove entry from our
480	* SW tables, because there's not much we can do about it.
481	*/
482	netif_err(efx, drv, efx->net_dev,
483	"Failed to release encap match %#x, rc %d\n",
484	encap->fw_id, rc);
485	}
486	rhashtable_remove_fast(ht: &efx->tc->encap_match_ht, obj: &encap->linkage,
487	params: efx_tc_encap_match_ht_params);
488	if (encap->pseudo)
489	efx_tc_flower_release_encap_match(efx, encap: encap->pseudo);
490	kfree(objp: encap);
491	}
492
493	static int efx_tc_flower_record_encap_match(struct efx_nic *efx,
494	struct efx_tc_match *match,
495	enum efx_encap_type type,
496	enum efx_tc_em_pseudo_type em_type,
497	u8 child_ip_tos_mask,
498	__be16 child_udp_sport_mask,
499	struct netlink_ext_ack *extack)
500	{
501	struct efx_tc_encap_match *encap, *old, *pseudo = NULL;
502	bool ipv6 = false;
503	int rc;
504
505	/* We require that the socket-defining fields (IP addrs and UDP dest
506	* port) are present and exact-match. Other fields may only be used
507	* if the field-set (and any masks) are the same for all encap
508	* matches on the same <sip,dip,dport> tuple; this is enforced by
509	* pseudo encap matches.
510	*/
511	if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {
512	if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {
513	NL_SET_ERR_MSG_MOD(extack,
514	"Egress encap match is not exact on dst IP address");
515	return -EOPNOTSUPP;
516	}
517	if (!IS_ALL_ONES(match->mask.enc_src_ip)) {
518	NL_SET_ERR_MSG_MOD(extack,
519	"Egress encap match is not exact on src IP address");
520	return -EOPNOTSUPP;
521	}
522	#ifdef CONFIG_IPV6
523	if (!ipv6_addr_any(a: &match->mask.enc_dst_ip6) ||
524	!ipv6_addr_any(a: &match->mask.enc_src_ip6)) {
525	NL_SET_ERR_MSG_MOD(extack,
526	"Egress encap match on both IPv4 and IPv6, don't understand");
527	return -EOPNOTSUPP;
528	}
529	} else {
530	ipv6 = true;
531	if (!efx_ipv6_addr_all_ones(addr: &match->mask.enc_dst_ip6)) {
532	NL_SET_ERR_MSG_MOD(extack,
533	"Egress encap match is not exact on dst IP address");
534	return -EOPNOTSUPP;
535	}
536	if (!efx_ipv6_addr_all_ones(addr: &match->mask.enc_src_ip6)) {
537	NL_SET_ERR_MSG_MOD(extack,
538	"Egress encap match is not exact on src IP address");
539	return -EOPNOTSUPP;
540	}
541	#endif
542	}
543	if (!IS_ALL_ONES(match->mask.enc_dport)) {
544	NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");
545	return -EOPNOTSUPP;
546	}
547	if (match->mask.enc_sport || match->mask.enc_ip_tos) {
548	struct efx_tc_match pmatch = *match;
549
550	if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */
551	NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler");
552	return -EOPNOTSUPP;
553	}
554	pmatch.value.enc_ip_tos = 0;
555	pmatch.mask.enc_ip_tos = 0;
556	pmatch.value.enc_sport = 0;
557	pmatch.mask.enc_sport = 0;
558	rc = efx_tc_flower_record_encap_match(efx, match: &pmatch, type,
559	em_type: EFX_TC_EM_PSEUDO_MASK,
560	child_ip_tos_mask: match->mask.enc_ip_tos,
561	child_udp_sport_mask: match->mask.enc_sport,
562	extack);
563	if (rc)
564	return rc;
565	pseudo = pmatch.encap;
566	}
567	if (match->mask.enc_ip_ttl) {
568	NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");
569	rc = -EOPNOTSUPP;
570	goto fail_pseudo;
571	}
572
573	rc = efx_mae_check_encap_match_caps(efx, ipv6, ip_tos_mask: match->mask.enc_ip_tos,
574	udp_sport_mask: match->mask.enc_sport, extack);
575	if (rc)
576	goto fail_pseudo;
577
578	encap = kzalloc(size: sizeof(*encap), GFP_USER);
579	if (!encap) {
580	rc = -ENOMEM;
581	goto fail_pseudo;
582	}
583	encap->src_ip = match->value.enc_src_ip;
584	encap->dst_ip = match->value.enc_dst_ip;
585	#ifdef CONFIG_IPV6
586	encap->src_ip6 = match->value.enc_src_ip6;
587	encap->dst_ip6 = match->value.enc_dst_ip6;
588	#endif
589	encap->udp_dport = match->value.enc_dport;
590	encap->tun_type = type;
591	encap->ip_tos = match->value.enc_ip_tos;
592	encap->ip_tos_mask = match->mask.enc_ip_tos;
593	encap->child_ip_tos_mask = child_ip_tos_mask;
594	encap->udp_sport = match->value.enc_sport;
595	encap->udp_sport_mask = match->mask.enc_sport;
596	encap->child_udp_sport_mask = child_udp_sport_mask;
597	encap->type = em_type;
598	encap->pseudo = pseudo;
599	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->encap_match_ht,
600	obj: &encap->linkage,
601	params: efx_tc_encap_match_ht_params);
602	if (old) {
603	/* don't need our new entry */
604	kfree(objp: encap);
605	if (pseudo) /* don't need our new pseudo either */
606	efx_tc_flower_release_encap_match(efx, encap: pseudo);
607	if (IS_ERR(ptr: old)) /* oh dear, it's actually an error */
608	return PTR_ERR(ptr: old);
609	/* check old and new em_types are compatible */
610	switch (old->type) {
611	case EFX_TC_EM_DIRECT:
612	/* old EM is in hardware, so mustn't overlap with a
613	* pseudo, but may be shared with another direct EM
614	*/
615	if (em_type == EFX_TC_EM_DIRECT)
616	break;
617	NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry");
618	return -EEXIST;
619	case EFX_TC_EM_PSEUDO_MASK:
620	/* old EM is protecting a ToS- or src port-qualified
621	* filter, so may only be shared with another pseudo
622	* for the same ToS and src port masks.
623	*/
624	if (em_type != EFX_TC_EM_PSEUDO_MASK) {
625	NL_SET_ERR_MSG_FMT_MOD(extack,
626	"%s encap match conflicts with existing pseudo(MASK) entry",
627	em_type ? "Pseudo" : "Direct");
628	return -EEXIST;
629	}
630	if (child_ip_tos_mask != old->child_ip_tos_mask) {
631	NL_SET_ERR_MSG_FMT_MOD(extack,
632	"Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x",
633	child_ip_tos_mask,
634	old->child_ip_tos_mask);
635	return -EEXIST;
636	}
637	if (child_udp_sport_mask != old->child_udp_sport_mask) {
638	NL_SET_ERR_MSG_FMT_MOD(extack,
639	"Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x",
640	child_udp_sport_mask,
641	old->child_udp_sport_mask);
642	return -EEXIST;
643	}
644	break;
645	case EFX_TC_EM_PSEUDO_OR:
646	/* old EM corresponds to an OR that has to be unique
647	* (it must not overlap with any other OR, whether
648	* direct-EM or pseudo).
649	*/
650	NL_SET_ERR_MSG_FMT_MOD(extack,
651	"%s encap match conflicts with existing pseudo(OR) entry",
652	em_type ? "Pseudo" : "Direct");
653	return -EEXIST;
654	default: /* Unrecognised pseudo-type. Just say no */
655	NL_SET_ERR_MSG_FMT_MOD(extack,
656	"%s encap match conflicts with existing pseudo(%d) entry",
657	em_type ? "Pseudo" : "Direct",
658	old->type);
659	return -EEXIST;
660	}
661	/* check old and new tun_types are compatible */
662	if (old->tun_type != type) {
663	NL_SET_ERR_MSG_FMT_MOD(extack,
664	"Egress encap match with conflicting tun_type %u != %u",
665	old->tun_type, type);
666	return -EEXIST;
667	}
668	if (!refcount_inc_not_zero(r: &old->ref))
669	return -EAGAIN;
670	/* existing entry found */
671	encap = old;
672	} else {
673	if (em_type == EFX_TC_EM_DIRECT) {
674	rc = efx_mae_register_encap_match(efx, encap);
675	if (rc) {
676	NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");
677	goto fail;
678	}
679	}
680	refcount_set(r: &encap->ref, n: 1);
681	}
682	match->encap = encap;
683	return 0;
684	fail:
685	rhashtable_remove_fast(ht: &efx->tc->encap_match_ht, obj: &encap->linkage,
686	params: efx_tc_encap_match_ht_params);
687	kfree(objp: encap);
688	fail_pseudo:
689	if (pseudo)
690	efx_tc_flower_release_encap_match(efx, encap: pseudo);
691	return rc;
692	}
693
694	static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx,
695	u32 chain_index,
696	struct net_device *net_dev)
697	{
698	struct efx_tc_recirc_id *rid, *old;
699	int rc;
700
701	rid = kzalloc(size: sizeof(*rid), GFP_USER);
702	if (!rid)
703	return ERR_PTR(error: -ENOMEM);
704	rid->chain_index = chain_index;
705	/* We don't take a reference here, because it's implied - if there's
706	* a rule on the net_dev that's been offloaded to us, then the net_dev
707	* can't go away until the rule has been deoffloaded.
708	*/
709	rid->net_dev = net_dev;
710	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->recirc_ht,
711	obj: &rid->linkage,
712	params: efx_tc_recirc_ht_params);
713	if (old) {
714	/* don't need our new entry */
715	kfree(objp: rid);
716	if (IS_ERR(ptr: old)) /* oh dear, it's actually an error */
717	return ERR_CAST(ptr: old);
718	if (!refcount_inc_not_zero(r: &old->ref))
719	return ERR_PTR(error: -EAGAIN);
720	/* existing entry found */
721	rid = old;
722	} else {
723	rc = ida_alloc_range(&efx->tc->recirc_ida, min: 1, U8_MAX, GFP_USER);
724	if (rc < 0) {
725	rhashtable_remove_fast(ht: &efx->tc->recirc_ht,
726	obj: &rid->linkage,
727	params: efx_tc_recirc_ht_params);
728	kfree(objp: rid);
729	return ERR_PTR(error: rc);
730	}
731	rid->fw_id = rc;
732	refcount_set(r: &rid->ref, n: 1);
733	}
734	return rid;
735	}
736
737	static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid)
738	{
739	if (!refcount_dec_and_test(r: &rid->ref))
740	return; /* still in use */
741	rhashtable_remove_fast(ht: &efx->tc->recirc_ht, obj: &rid->linkage,
742	params: efx_tc_recirc_ht_params);
743	ida_free(&efx->tc->recirc_ida, id: rid->fw_id);
744	kfree(objp: rid);
745	}
746
747	static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
748	{
749	efx_mae_delete_rule(efx, id: rule->fw_id);
750
751	/* Release entries in subsidiary tables */
752	efx_tc_free_action_set_list(efx, acts: &rule->acts, in_hw: true);
753	if (rule->match.rid)
754	efx_tc_put_recirc_id(efx, rid: rule->match.rid);
755	if (rule->match.encap)
756	efx_tc_flower_release_encap_match(efx, encap: rule->match.encap);
757	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
758	}
759
760	static const char *efx_tc_encap_type_name(enum efx_encap_type typ)
761	{
762	switch (typ) {
763	case EFX_ENCAP_TYPE_NONE:
764	return "none";
765	case EFX_ENCAP_TYPE_VXLAN:
766	return "vxlan";
767	case EFX_ENCAP_TYPE_GENEVE:
768	return "geneve";
769	default:
770	pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);
771	return "unknown";
772	}
773	}
774
775	/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
776	enum efx_tc_action_order {
777	EFX_TC_AO_DECAP,
778	EFX_TC_AO_DEC_TTL,
779	EFX_TC_AO_PEDIT_MAC_ADDRS,
780	EFX_TC_AO_VLAN_POP,
781	EFX_TC_AO_VLAN_PUSH,
782	EFX_TC_AO_COUNT,
783	EFX_TC_AO_ENCAP,
784	EFX_TC_AO_DELIVER
785	};
786	/* Determine whether we can add @new action without violating order */
787	static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
788	enum efx_tc_action_order new)
789	{
790	switch (new) {
791	case EFX_TC_AO_DECAP:
792	if (act->decap)
793	return false;
794	/* PEDIT_MAC_ADDRS must not happen before DECAP, though it
795	* can wait until much later
796	*/
797	if (act->dst_mac || act->src_mac)
798	return false;
799
800	/* Decrementing ttl must not happen before DECAP */
801	if (act->do_ttl_dec)
802	return false;
803	fallthrough;
804	case EFX_TC_AO_VLAN_POP:
805	if (act->vlan_pop >= 2)
806	return false;
807	/* If we've already pushed a VLAN, we can't then pop it;
808	* the hardware would instead try to pop an existing VLAN
809	* before pushing the new one.
810	*/
811	if (act->vlan_push)
812	return false;
813	fallthrough;
814	case EFX_TC_AO_VLAN_PUSH:
815	if (act->vlan_push >= 2)
816	return false;
817	fallthrough;
818	case EFX_TC_AO_COUNT:
819	if (act->count)
820	return false;
821	fallthrough;
822	case EFX_TC_AO_PEDIT_MAC_ADDRS:
823	case EFX_TC_AO_ENCAP:
824	if (act->encap_md)
825	return false;
826	fallthrough;
827	case EFX_TC_AO_DELIVER:
828	return !act->deliver;
829	case EFX_TC_AO_DEC_TTL:
830	if (act->encap_md)
831	return false;
832	return !act->do_ttl_dec;
833	default:
834	/* Bad caller. Whatever they wanted to do, say they can't. */
835	WARN_ON_ONCE(1);
836	return false;
837	}
838	}
839
840	/**
841	* DOC: TC conntrack sequences
842	*
843	* The MAE hardware can handle at most two rounds of action rule matching,
844	* consequently we support conntrack through the notion of a "left-hand side
845	* rule". This is a rule which typically contains only the actions "ct" and
846	* "goto chain N", and corresponds to one or more "right-hand side rules" in
847	* chain N, which typically match on +trk+est, and may perform ct(nat) actions.
848	* RHS rules go in the Action Rule table as normal but with a nonzero recirc_id
849	* (the hardware equivalent of chain_index), while LHS rules may go in either
850	* the Action Rule or the Outer Rule table, the latter being preferred for
851	* performance reasons, and set both DO_CT and a recirc_id in their response.
852	*
853	* Besides the RHS rules, there are often also similar rules matching on
854	* +trk+new which perform the ct(commit) action. These are not offloaded.
855	*/
856
857	static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr,
858	struct efx_tc_match *match)
859	{
860	const struct flow_action_entry *fa;
861	int i;
862
863	flow_action_for_each(i, fa, &fr->action) {
864	switch (fa->id) {
865	case FLOW_ACTION_GOTO:
866	return true;
867	case FLOW_ACTION_CT:
868	/* If rule is -trk, or doesn't mention trk at all, then
869	* a CT action implies a conntrack lookup (hence it's an
870	* LHS rule). If rule is +trk, then a CT action could
871	* just be ct(nat) or even ct(commit) (though the latter
872	* can't be offloaded).
873	*/
874	if (!match->mask.ct_state_trk || !match->value.ct_state_trk)
875	return true;
876	break;
877	default:
878	break;
879	}
880	}
881	return false;
882	}
883
884	/* A foreign LHS rule has matches on enc_ keys at the TC layer (including an
885	* implied match on enc_ip_proto UDP). Translate these into non-enc_ keys,
886	* so that we can use the same MAE machinery as local LHS rules (and so that
887	* the lhs_rules entries have uniform semantics). It may seem odd to do it
888	* this way round, given that the corresponding fields in the MAE MCDIs are
889	* all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct
890	* efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner
891	* fields so it's just matching on *the* header rather than the outer header.
892	* Make sure that the non-enc_ keys were not already being matched on, as that
893	* would imply a rule that needed a triple lookup. (Hardware can do that,
894	* with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;
895	* see efx_tc_flower_flhs_needs_ar().)
896	*/
897	static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)
898	{
899	int rc = 0;
900
901	#define COPY_MASK_AND_VALUE(_key, _ekey) ({ \
902	if (match->mask._key) { \
903	rc = -EOPNOTSUPP; \
904	} else { \
905	match->mask._key = match->mask._ekey; \
906	match->mask._ekey = 0; \
907	match->value._key = match->value._ekey; \
908	match->value._ekey = 0; \
909	} \
910	rc; \
911	})
912	#define COPY_FROM_ENC(_key) COPY_MASK_AND_VALUE(_key, enc_##_key)
913	if (match->mask.ip_proto)
914	return -EOPNOTSUPP;
915	match->mask.ip_proto = ~0;
916	match->value.ip_proto = IPPROTO_UDP;
917	if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip))
918	return rc;
919	#ifdef CONFIG_IPV6
920	if (!ipv6_addr_any(a: &match->mask.src_ip6))
921	return -EOPNOTSUPP;
922	match->mask.src_ip6 = match->mask.enc_src_ip6;
923	memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));
924	if (!ipv6_addr_any(a: &match->mask.dst_ip6))
925	return -EOPNOTSUPP;
926	match->mask.dst_ip6 = match->mask.enc_dst_ip6;
927	memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));
928	#endif
929	if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl))
930	return rc;
931	/* should really copy enc_ip_frag but we don't have that in
932	* parse_match yet
933	*/
934	if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) ||
935	COPY_MASK_AND_VALUE(l4_dport, enc_dport))
936	return rc;
937	return 0;
938	#undef COPY_FROM_ENC
939	#undef COPY_MASK_AND_VALUE
940	}
941
942	/* If a foreign LHS rule wants to match on keys that are only available after
943	* encap header identification and parsing, then it can't be done in the Outer
944	* Rule lookup, because that lookup determines the encap type used to parse
945	* beyond the outer headers. Thus, such rules must use the OR-AR-CT-AR lookup
946	* sequence, with an EM (struct efx_tc_encap_match) in the OR step.
947	* Return true iff the passed match requires this.
948	*/
949	static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)
950	{
951	/* matches on inner-header keys can't be done in OR */
952	return match->mask.eth_proto ||
953	match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||
954	match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||
955	memchr_inv(p: match->mask.eth_saddr, c: 0, ETH_ALEN) ||
956	memchr_inv(p: match->mask.eth_daddr, c: 0, ETH_ALEN) ||
957	match->mask.ip_proto ||
958	match->mask.ip_tos || match->mask.ip_ttl ||
959	match->mask.src_ip || match->mask.dst_ip ||
960	#ifdef CONFIG_IPV6
961	!ipv6_addr_any(a: &match->mask.src_ip6) ||
962	!ipv6_addr_any(a: &match->mask.dst_ip6) ||
963	#endif
964	match->mask.ip_frag || match->mask.ip_firstfrag ||
965	match->mask.l4_sport || match->mask.l4_dport ||
966	match->mask.tcp_flags ||
967	/* nor can VNI */
968	match->mask.enc_keyid;
969	}
970
971	static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx,
972	struct flow_cls_offload *tc,
973	struct flow_rule *fr,
974	struct net_device *net_dev,
975	struct efx_tc_lhs_rule *rule)
976
977	{
978	struct netlink_ext_ack *extack = tc->common.extack;
979	struct efx_tc_lhs_action *act = &rule->lhs_act;
980	const struct flow_action_entry *fa;
981	enum efx_tc_counter_type ctype;
982	bool pipe = true;
983	int i;
984
985	ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;
986
987	flow_action_for_each(i, fa, &fr->action) {
988	struct efx_tc_ct_zone *ct_zone;
989	struct efx_tc_recirc_id *rid;
990
991	if (!pipe) {
992	/* more actions after a non-pipe action */
993	NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
994	return -EINVAL;
995	}
996	switch (fa->id) {
997	case FLOW_ACTION_GOTO:
998	if (!fa->chain_index) {
999	NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw");
1000	return -EOPNOTSUPP;
1001	}
1002	rid = efx_tc_get_recirc_id(efx, chain_index: fa->chain_index,
1003	net_dev);
1004	if (IS_ERR(ptr: rid)) {
1005	NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index");
1006	return PTR_ERR(ptr: rid);
1007	}
1008	act->rid = rid;
1009	if (fa->hw_stats) {
1010	struct efx_tc_counter_index *cnt;
1011
1012	if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1013	NL_SET_ERR_MSG_FMT_MOD(extack,
1014	"hw_stats_type %u not supported (only 'delayed')",
1015	fa->hw_stats);
1016	return -EOPNOTSUPP;
1017	}
1018	cnt = efx_tc_flower_get_counter_index(efx, cookie: tc->cookie,
1019	type: ctype);
1020	if (IS_ERR(ptr: cnt)) {
1021	NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1022	return PTR_ERR(ptr: cnt);
1023	}
1024	WARN_ON(act->count); /* can't happen */
1025	act->count = cnt;
1026	}
1027	pipe = false;
1028	break;
1029	case FLOW_ACTION_CT:
1030	if (act->zone) {
1031	NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions");
1032	return -EOPNOTSUPP;
1033	}
1034	if (fa->ct.action & (TCA_CT_ACT_COMMIT |
1035	TCA_CT_ACT_FORCE)) {
1036	NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force");
1037	return -EOPNOTSUPP;
1038	}
1039	if (fa->ct.action & TCA_CT_ACT_CLEAR) {
1040	NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule");
1041	return -EOPNOTSUPP;
1042	}
1043	if (fa->ct.action & (TCA_CT_ACT_NAT |
1044	TCA_CT_ACT_NAT_SRC |
1045	TCA_CT_ACT_NAT_DST)) {
1046	NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet");
1047	return -EOPNOTSUPP;
1048	}
1049	if (fa->ct.action) {
1050	NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n",
1051	fa->ct.action);
1052	return -EOPNOTSUPP;
1053	}
1054	ct_zone = efx_tc_ct_register_zone(efx, zone: fa->ct.zone,
1055	ct_ft: fa->ct.flow_table);
1056	if (IS_ERR(ptr: ct_zone)) {
1057	NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates");
1058	return PTR_ERR(ptr: ct_zone);
1059	}
1060	act->zone = ct_zone;
1061	break;
1062	default:
1063	NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n",
1064	fa->id);
1065	return -EOPNOTSUPP;
1066	}
1067	}
1068
1069	if (pipe) {
1070	NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule");
1071	return -EOPNOTSUPP;
1072	}
1073	return 0;
1074	}
1075
1076	static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx,
1077	struct efx_tc_lhs_action *act)
1078	{
1079	if (act->rid)
1080	efx_tc_put_recirc_id(efx, rid: act->rid);
1081	if (act->zone)
1082	efx_tc_ct_unregister_zone(efx, ct_zone: act->zone);
1083	if (act->count)
1084	efx_tc_flower_put_counter_index(efx, ctr: act->count);
1085	}
1086
1087	/**
1088	* struct efx_tc_mangler_state - accumulates 32-bit pedits into fields
1089	*
1090	* @dst_mac_32: dst_mac[0:3] has been populated
1091	* @dst_mac_16: dst_mac[4:5] has been populated
1092	* @src_mac_16: src_mac[0:1] has been populated
1093	* @src_mac_32: src_mac[2:5] has been populated
1094	* @dst_mac: h_dest field of ethhdr
1095	* @src_mac: h_source field of ethhdr
1096	*
1097	* Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not
1098	* necessarily equate to whole fields of the packet header, this
1099	* structure is used to hold the cumulative effect of the partial
1100	* field pedits that have been processed so far.
1101	*/
1102	struct efx_tc_mangler_state {
1103	u8 dst_mac_32:1; /* eth->h_dest[0:3] */
1104	u8 dst_mac_16:1; /* eth->h_dest[4:5] */
1105	u8 src_mac_16:1; /* eth->h_source[0:1] */
1106	u8 src_mac_32:1; /* eth->h_source[2:5] */
1107	unsigned char dst_mac[ETH_ALEN];
1108	unsigned char src_mac[ETH_ALEN];
1109	};
1110
1111	/** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung
1112	* @efx: NIC we're installing a flow rule on
1113	* @act: action set (cursor) to update
1114	* @mung: accumulated partial mangles
1115	* @extack: netlink extended ack for reporting errors
1116	*
1117	* Check @mung to find any combinations of partial mangles that can be
1118	* combined into a complete packet field edit, add that edit to @act,
1119	* and consume the partial mangles from @mung.
1120	*/
1121
1122	static int efx_tc_complete_mac_mangle(struct efx_nic *efx,
1123	struct efx_tc_action_set *act,
1124	struct efx_tc_mangler_state *mung,
1125	struct netlink_ext_ack *extack)
1126	{
1127	struct efx_tc_mac_pedit_action *ped;
1128
1129	if (mung->dst_mac_32 && mung->dst_mac_16) {
1130	ped = efx_tc_flower_get_mac(efx, h_addr: mung->dst_mac, extack);
1131	if (IS_ERR(ptr: ped))
1132	return PTR_ERR(ptr: ped);
1133
1134	/* Check that we have not already populated dst_mac */
1135	if (act->dst_mac)
1136	efx_tc_flower_put_mac(efx, ped: act->dst_mac);
1137
1138	act->dst_mac = ped;
1139
1140	/* consume the incomplete state */
1141	mung->dst_mac_32 = 0;
1142	mung->dst_mac_16 = 0;
1143	}
1144	if (mung->src_mac_16 && mung->src_mac_32) {
1145	ped = efx_tc_flower_get_mac(efx, h_addr: mung->src_mac, extack);
1146	if (IS_ERR(ptr: ped))
1147	return PTR_ERR(ptr: ped);
1148
1149	/* Check that we have not already populated src_mac */
1150	if (act->src_mac)
1151	efx_tc_flower_put_mac(efx, ped: act->src_mac);
1152
1153	act->src_mac = ped;
1154
1155	/* consume the incomplete state */
1156	mung->src_mac_32 = 0;
1157	mung->src_mac_16 = 0;
1158	}
1159	return 0;
1160	}
1161
1162	static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act,
1163	const struct flow_action_entry *fa,
1164	struct netlink_ext_ack *extack)
1165	{
1166	switch (fa->mangle.htype) {
1167	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1168	switch (fa->mangle.offset) {
1169	case offsetof(struct iphdr, ttl):
1170	/* check that pedit applies to ttl only */
1171	if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK)
1172	break;
1173
1174	/* Adding 0xff is equivalent to decrementing the ttl.
1175	* Other added values are not supported.
1176	*/
1177	if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX)
1178	break;
1179
1180	/* check that we do not decrement ttl twice */
1181	if (!efx_tc_flower_action_order_ok(act,
1182	new: EFX_TC_AO_DEC_TTL)) {
1183	NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1184	return -EOPNOTSUPP;
1185	}
1186	act->do_ttl_dec = 1;
1187	return 0;
1188	default:
1189	break;
1190	}
1191	break;
1192	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1193	switch (fa->mangle.offset) {
1194	case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1195	/* check that pedit applies to hoplimit only */
1196	if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK)
1197	break;
1198
1199	/* Adding 0xff is equivalent to decrementing the hoplimit.
1200	* Other added values are not supported.
1201	*/
1202	if ((fa->mangle.val >> 24) != U8_MAX)
1203	break;
1204
1205	/* check that we do not decrement hoplimit twice */
1206	if (!efx_tc_flower_action_order_ok(act,
1207	new: EFX_TC_AO_DEC_TTL)) {
1208	NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1209	return -EOPNOTSUPP;
1210	}
1211	act->do_ttl_dec = 1;
1212	return 0;
1213	default:
1214	break;
1215	}
1216	break;
1217	default:
1218	break;
1219	}
1220
1221	NL_SET_ERR_MSG_FMT_MOD(extack,
1222	"ttl add action type %x %x %x/%x is not supported",
1223	fa->mangle.htype, fa->mangle.offset,
1224	fa->mangle.val, fa->mangle.mask);
1225	return -EOPNOTSUPP;
1226	}
1227
1228	/**
1229	* efx_tc_mangle() - handle a single 32-bit (or less) pedit
1230	* @efx: NIC we're installing a flow rule on
1231	* @act: action set (cursor) to update
1232	* @fa: FLOW_ACTION_MANGLE action metadata
1233	* @mung: accumulator for partial mangles
1234	* @extack: netlink extended ack for reporting errors
1235	* @match: original match used along with the mangle action
1236	*
1237	* Identify the fields written by a FLOW_ACTION_MANGLE, and record
1238	* the partial mangle state in @mung. If this mangle completes an
1239	* earlier partial mangle, consume and apply to @act by calling
1240	* efx_tc_complete_mac_mangle().
1241	*/
1242
1243	static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act,
1244	const struct flow_action_entry *fa,
1245	struct efx_tc_mangler_state *mung,
1246	struct netlink_ext_ack *extack,
1247	struct efx_tc_match *match)
1248	{
1249	__le32 mac32;
1250	__le16 mac16;
1251	u8 tr_ttl;
1252
1253	switch (fa->mangle.htype) {
1254	case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
1255	BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0);
1256	BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6);
1257	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_PEDIT_MAC_ADDRS)) {
1258	NL_SET_ERR_MSG_MOD(extack,
1259	"Pedit mangle mac action violates action order");
1260	return -EOPNOTSUPP;
1261	}
1262	switch (fa->mangle.offset) {
1263	case 0:
1264	if (fa->mangle.mask) {
1265	NL_SET_ERR_MSG_FMT_MOD(extack,
1266	"mask (%#x) of eth.dst32 mangle is not supported",
1267	fa->mangle.mask);
1268	return -EOPNOTSUPP;
1269	}
1270	/* Ethernet address is little-endian */
1271	mac32 = cpu_to_le32(fa->mangle.val);
1272	memcpy(mung->dst_mac, &mac32, sizeof(mac32));
1273	mung->dst_mac_32 = 1;
1274	return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1275	case 4:
1276	if (fa->mangle.mask == 0xffff) {
1277	mac16 = cpu_to_le16(fa->mangle.val >> 16);
1278	memcpy(mung->src_mac, &mac16, sizeof(mac16));
1279	mung->src_mac_16 = 1;
1280	} else if (fa->mangle.mask == 0xffff0000) {
1281	mac16 = cpu_to_le16((u16)fa->mangle.val);
1282	memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16));
1283	mung->dst_mac_16 = 1;
1284	} else {
1285	NL_SET_ERR_MSG_FMT_MOD(extack,
1286	"mask (%#x) of eth+4 mangle is not high or low 16b",
1287	fa->mangle.mask);
1288	return -EOPNOTSUPP;
1289	}
1290	return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1291	case 8:
1292	if (fa->mangle.mask) {
1293	NL_SET_ERR_MSG_FMT_MOD(extack,
1294	"mask (%#x) of eth.src32 mangle is not supported",
1295	fa->mangle.mask);
1296	return -EOPNOTSUPP;
1297	}
1298	mac32 = cpu_to_le32(fa->mangle.val);
1299	memcpy(mung->src_mac + 2, &mac32, sizeof(mac32));
1300	mung->src_mac_32 = 1;
1301	return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1302	default:
1303	NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported",
1304	fa->mangle.offset, fa->mangle.val, fa->mangle.mask);
1305	return -EOPNOTSUPP;
1306	}
1307	break;
1308	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1309	switch (fa->mangle.offset) {
1310	case offsetof(struct iphdr, ttl):
1311	/* we currently only support pedit IP4 when it applies
1312	* to TTL and then only when it can be achieved with a
1313	* decrement ttl action
1314	*/
1315
1316	/* check that pedit applies to ttl only */
1317	if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) {
1318	NL_SET_ERR_MSG_FMT_MOD(extack,
1319	"mask (%#x) out of range, only support mangle action on ipv4.ttl",
1320	fa->mangle.mask);
1321	return -EOPNOTSUPP;
1322	}
1323
1324	/* we can only convert to a dec ttl when we have an
1325	* exact match on the ttl field
1326	*/
1327	if (match->mask.ip_ttl != U8_MAX) {
1328	NL_SET_ERR_MSG_FMT_MOD(extack,
1329	"only support mangle ttl when we have an exact match, current mask (%#x)",
1330	match->mask.ip_ttl);
1331	return -EOPNOTSUPP;
1332	}
1333
1334	/* check that we don't try to decrement 0, which equates
1335	* to setting the ttl to 0xff
1336	*/
1337	if (match->value.ip_ttl == 0) {
1338	NL_SET_ERR_MSG_MOD(extack,
1339	"decrement ttl past 0 is not supported");
1340	return -EOPNOTSUPP;
1341	}
1342
1343	/* check that we do not decrement ttl twice */
1344	if (!efx_tc_flower_action_order_ok(act,
1345	new: EFX_TC_AO_DEC_TTL)) {
1346	NL_SET_ERR_MSG_MOD(extack,
1347	"multiple dec ttl is not supported");
1348	return -EOPNOTSUPP;
1349	}
1350
1351	/* check pedit can be achieved with decrement action */
1352	tr_ttl = match->value.ip_ttl - 1;
1353	if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) {
1354	act->do_ttl_dec = 1;
1355	return 0;
1356	}
1357
1358	fallthrough;
1359	default:
1360	NL_SET_ERR_MSG_FMT_MOD(extack,
1361	"only support mangle on the ttl field (offset is %u)",
1362	fa->mangle.offset);
1363	return -EOPNOTSUPP;
1364	}
1365	break;
1366	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1367	switch (fa->mangle.offset) {
1368	case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1369	/* we currently only support pedit IP6 when it applies
1370	* to the hoplimit and then only when it can be achieved
1371	* with a decrement hoplimit action
1372	*/
1373
1374	/* check that pedit applies to ttl only */
1375	if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) {
1376	NL_SET_ERR_MSG_FMT_MOD(extack,
1377	"mask (%#x) out of range, only support mangle action on ipv6.hop_limit",
1378	fa->mangle.mask);
1379
1380	return -EOPNOTSUPP;
1381	}
1382
1383	/* we can only convert to a dec ttl when we have an
1384	* exact match on the ttl field
1385	*/
1386	if (match->mask.ip_ttl != U8_MAX) {
1387	NL_SET_ERR_MSG_FMT_MOD(extack,
1388	"only support hop_limit when we have an exact match, current mask (%#x)",
1389	match->mask.ip_ttl);
1390	return -EOPNOTSUPP;
1391	}
1392
1393	/* check that we don't try to decrement 0, which equates
1394	* to setting the ttl to 0xff
1395	*/
1396	if (match->value.ip_ttl == 0) {
1397	NL_SET_ERR_MSG_MOD(extack,
1398	"decrementing hop_limit past 0 is not supported");
1399	return -EOPNOTSUPP;
1400	}
1401
1402	/* check that we do not decrement hoplimit twice */
1403	if (!efx_tc_flower_action_order_ok(act,
1404	new: EFX_TC_AO_DEC_TTL)) {
1405	NL_SET_ERR_MSG_MOD(extack,
1406	"multiple dec ttl is not supported");
1407	return -EOPNOTSUPP;
1408	}
1409
1410	/* check pedit can be achieved with decrement action */
1411	tr_ttl = match->value.ip_ttl - 1;
1412	if ((fa->mangle.val >> 24) == tr_ttl) {
1413	act->do_ttl_dec = 1;
1414	return 0;
1415	}
1416
1417	fallthrough;
1418	default:
1419	NL_SET_ERR_MSG_FMT_MOD(extack,
1420	"only support mangle on the hop_limit field");
1421	return -EOPNOTSUPP;
1422	}
1423	default:
1424	NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule",
1425	fa->mangle.htype);
1426	return -EOPNOTSUPP;
1427	}
1428	return 0;
1429	}
1430
1431	/**
1432	* efx_tc_incomplete_mangle() - check for leftover partial pedits
1433	* @mung: accumulator for partial mangles
1434	* @extack: netlink extended ack for reporting errors
1435	*
1436	* Since the MAE can only overwrite whole fields, any partial
1437	* field mangle left over on reaching packet delivery (mirred or
1438	* end of TC actions) cannot be offloaded. Check for any such
1439	* and reject them with -%EOPNOTSUPP.
1440	*/
1441
1442	static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung,
1443	struct netlink_ext_ack *extack)
1444	{
1445	if (mung->dst_mac_32 || mung->dst_mac_16) {
1446	NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address");
1447	return -EOPNOTSUPP;
1448	}
1449	if (mung->src_mac_16 || mung->src_mac_32) {
1450	NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address");
1451	return -EOPNOTSUPP;
1452	}
1453	return 0;
1454	}
1455
1456	static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx,
1457	struct flow_cls_offload *tc,
1458	struct flow_rule *fr,
1459	struct efx_tc_match *match,
1460	struct net_device *net_dev)
1461	{
1462	struct netlink_ext_ack *extack = tc->common.extack;
1463	struct efx_tc_lhs_rule *rule, *old;
1464	enum efx_encap_type type;
1465	int rc;
1466
1467	type = efx_tc_indr_netdev_type(net_dev);
1468	if (type == EFX_ENCAP_TYPE_NONE) {
1469	NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device");
1470	return -EOPNOTSUPP;
1471	}
1472
1473	rc = efx_mae_check_encap_type_supported(efx, typ: type);
1474	if (rc) {
1475	NL_SET_ERR_MSG_FMT_MOD(extack,
1476	"Firmware reports no support for %s encap match",
1477	efx_tc_encap_type_name(type));
1478	return rc;
1479	}
1480	/* This is an Action Rule, so it needs a separate Encap Match in the
1481	* Outer Rule table. Insert that now.
1482	*/
1483	rc = efx_tc_flower_record_encap_match(efx, match, type,
1484	em_type: EFX_TC_EM_DIRECT, child_ip_tos_mask: 0, child_udp_sport_mask: 0, extack);
1485	if (rc)
1486	return rc;
1487
1488	match->mask.recirc_id = 0xff;
1489	if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1490	NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1491	rc = -EOPNOTSUPP;
1492	goto release_encap_match;
1493	}
1494	/* LHS rules are always -trk, so we don't need to match on that */
1495	match->mask.ct_state_trk = 0;
1496	match->value.ct_state_trk = 0;
1497	/* We must inhibit match on TCP SYN/FIN/RST, so that SW can see
1498	* the packet and update the conntrack table.
1499	* Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action
1500	* Rules don't have that; instead they support matching on
1501	* TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that.
1502	* This is only strictly needed if there will be a DO_CT action,
1503	* which we don't know yet, but typically there will be and it's
1504	* simpler not to bother checking here.
1505	*/
1506	match->mask.tcp_syn_fin_rst = true;
1507
1508	rc = efx_mae_match_check_caps(efx, mask: &match->mask, extack);
1509	if (rc)
1510	goto release_encap_match;
1511
1512	rule = kzalloc(size: sizeof(*rule), GFP_USER);
1513	if (!rule) {
1514	rc = -ENOMEM;
1515	goto release_encap_match;
1516	}
1517	rule->cookie = tc->cookie;
1518	rule->is_ar = true;
1519	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->lhs_rule_ht,
1520	obj: &rule->linkage,
1521	params: efx_tc_lhs_rule_ht_params);
1522	if (old) {
1523	netif_dbg(efx, drv, efx->net_dev,
1524	"Already offloaded rule (cookie %lx)\n", tc->cookie);
1525	rc = -EEXIST;
1526	NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1527	goto release;
1528	}
1529
1530	/* Parse actions */
1531	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1532	if (rc)
1533	goto release;
1534
1535	rule->match = *match;
1536	rule->lhs_act.tun_type = type;
1537
1538	rc = efx_mae_insert_lhs_rule(efx, rule, prio: EFX_TC_PRIO_TC);
1539	if (rc) {
1540	NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1541	goto release;
1542	}
1543	netif_dbg(efx, drv, efx->net_dev,
1544	"Successfully parsed lhs rule (cookie %lx)\n",
1545	tc->cookie);
1546	return 0;
1547
1548	release:
1549	efx_tc_flower_release_lhs_actions(efx, act: &rule->lhs_act);
1550	if (!old)
1551	rhashtable_remove_fast(ht: &efx->tc->lhs_rule_ht, obj: &rule->linkage,
1552	params: efx_tc_lhs_rule_ht_params);
1553	kfree(objp: rule);
1554	release_encap_match:
1555	if (match->encap)
1556	efx_tc_flower_release_encap_match(efx, encap: match->encap);
1557	return rc;
1558	}
1559
1560	static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx,
1561	struct flow_cls_offload *tc,
1562	struct flow_rule *fr,
1563	struct efx_tc_match *match,
1564	struct net_device *net_dev)
1565	{
1566	struct netlink_ext_ack *extack = tc->common.extack;
1567	struct efx_tc_lhs_rule *rule, *old;
1568	enum efx_encap_type type;
1569	int rc;
1570
1571	if (tc->common.chain_index) {
1572	NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
1573	return -EOPNOTSUPP;
1574	}
1575
1576	if (!efx_tc_match_is_encap(mask: &match->mask)) {
1577	/* This is not a tunnel decap rule, ignore it */
1578	netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n");
1579	return -EOPNOTSUPP;
1580	}
1581
1582	if (efx_tc_flower_flhs_needs_ar(match))
1583	return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,
1584	net_dev);
1585
1586	type = efx_tc_indr_netdev_type(net_dev);
1587	if (type == EFX_ENCAP_TYPE_NONE) {
1588	NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device\n");
1589	return -EOPNOTSUPP;
1590	}
1591
1592	rc = efx_mae_check_encap_type_supported(efx, typ: type);
1593	if (rc) {
1594	NL_SET_ERR_MSG_FMT_MOD(extack,
1595	"Firmware reports no support for %s encap match",
1596	efx_tc_encap_type_name(type));
1597	return rc;
1598	}
1599	/* Reserve the outer tuple with a pseudo Encap Match */
1600	rc = efx_tc_flower_record_encap_match(efx, match, type,
1601	em_type: EFX_TC_EM_PSEUDO_OR, child_ip_tos_mask: 0, child_udp_sport_mask: 0,
1602	extack);
1603	if (rc)
1604	return rc;
1605
1606	if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1607	NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1608	rc = -EOPNOTSUPP;
1609	goto release_encap_match;
1610	}
1611	/* LHS rules are always -trk, so we don't need to match on that */
1612	match->mask.ct_state_trk = 0;
1613	match->value.ct_state_trk = 0;
1614
1615	rc = efx_tc_flower_translate_flhs_match(match);
1616	if (rc) {
1617	NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");
1618	goto release_encap_match;
1619	}
1620
1621	rc = efx_mae_match_check_caps_lhs(efx, mask: &match->mask, extack);
1622	if (rc)
1623	goto release_encap_match;
1624
1625	rule = kzalloc(size: sizeof(*rule), GFP_USER);
1626	if (!rule) {
1627	rc = -ENOMEM;
1628	goto release_encap_match;
1629	}
1630	rule->cookie = tc->cookie;
1631	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->lhs_rule_ht,
1632	obj: &rule->linkage,
1633	params: efx_tc_lhs_rule_ht_params);
1634	if (old) {
1635	netif_dbg(efx, drv, efx->net_dev,
1636	"Already offloaded rule (cookie %lx)\n", tc->cookie);
1637	rc = -EEXIST;
1638	NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1639	goto release;
1640	}
1641
1642	/* Parse actions */
1643	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1644	if (rc)
1645	goto release;
1646
1647	rule->match = *match;
1648	rule->lhs_act.tun_type = type;
1649
1650	rc = efx_mae_insert_lhs_rule(efx, rule, prio: EFX_TC_PRIO_TC);
1651	if (rc) {
1652	NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1653	goto release;
1654	}
1655	netif_dbg(efx, drv, efx->net_dev,
1656	"Successfully parsed lhs rule (cookie %lx)\n",
1657	tc->cookie);
1658	return 0;
1659
1660	release:
1661	efx_tc_flower_release_lhs_actions(efx, act: &rule->lhs_act);
1662	if (!old)
1663	rhashtable_remove_fast(ht: &efx->tc->lhs_rule_ht, obj: &rule->linkage,
1664	params: efx_tc_lhs_rule_ht_params);
1665	kfree(objp: rule);
1666	release_encap_match:
1667	if (match->encap)
1668	efx_tc_flower_release_encap_match(efx, encap: match->encap);
1669	return rc;
1670	}
1671
1672	static int efx_tc_flower_replace_foreign(struct efx_nic *efx,
1673	struct net_device *net_dev,
1674	struct flow_cls_offload *tc)
1675	{
1676	struct flow_rule *fr = flow_cls_offload_flow_rule(flow_cmd: tc);
1677	struct netlink_ext_ack *extack = tc->common.extack;
1678	struct efx_tc_flow_rule *rule = NULL, *old = NULL;
1679	struct efx_tc_action_set *act = NULL;
1680	bool found = false, uplinked = false;
1681	const struct flow_action_entry *fa;
1682	struct efx_tc_match match;
1683	struct efx_rep *to_efv;
1684	s64 rc;
1685	int i;
1686
1687	/* Parse match */
1688	memset(&match, 0, sizeof(match));
1689	rc = efx_tc_flower_parse_match(efx, rule: fr, match: &match, extack);
1690	if (rc)
1691	return rc;
1692	/* The rule as given to us doesn't specify a source netdevice.
1693	* But, determining whether packets from a VF should match it is
1694	* complicated, so leave those to the software slowpath: qualify
1695	* the filter with source m-port == wire.
1696	*/
1697	rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);
1698	if (rc < 0) {
1699	NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");
1700	return rc;
1701	}
1702	match.value.ingress_port = rc;
1703	match.mask.ingress_port = ~0;
1704
1705	if (efx_tc_rule_is_lhs_rule(fr, match: &match))
1706	return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, match: &match,
1707	net_dev);
1708
1709	if (tc->common.chain_index) {
1710	struct efx_tc_recirc_id *rid;
1711
1712	rid = efx_tc_get_recirc_id(efx, chain_index: tc->common.chain_index, net_dev);
1713	if (IS_ERR(ptr: rid)) {
1714	NL_SET_ERR_MSG_FMT_MOD(extack,
1715	"Failed to allocate a hardware recirculation ID for chain_index %u",
1716	tc->common.chain_index);
1717	return PTR_ERR(ptr: rid);
1718	}
1719	match.rid = rid;
1720	match.value.recirc_id = rid->fw_id;
1721	}
1722	match.mask.recirc_id = 0xff;
1723
1724	/* AR table can't match on DO_CT (+trk). But a commonly used pattern is
1725	* +trk+est, which is strictly implied by +est, so rewrite it to that.
1726	*/
1727	if (match.mask.ct_state_trk && match.value.ct_state_trk &&
1728	match.mask.ct_state_est && match.value.ct_state_est)
1729	match.mask.ct_state_trk = 0;
1730	/* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
1731	* match +trk-est (CT_HIT=0) despite being on an established connection.
1732	* So make -est imply -tcp_syn_fin_rst match to ensure these packets
1733	* still hit the software path.
1734	*/
1735	if (match.mask.ct_state_est && !match.value.ct_state_est) {
1736	if (match.value.tcp_syn_fin_rst) {
1737	/* Can't offload this combination */
1738	NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload");
1739	rc = -EOPNOTSUPP;
1740	goto release;
1741	}
1742	match.mask.tcp_syn_fin_rst = true;
1743	}
1744
1745	flow_action_for_each(i, fa, &fr->action) {
1746	switch (fa->id) {
1747	case FLOW_ACTION_REDIRECT:
1748	case FLOW_ACTION_MIRRED: /* mirred means mirror here */
1749	to_efv = efx_tc_flower_lookup_efv(efx, dev: fa->dev);
1750	if (IS_ERR(ptr: to_efv))
1751	continue;
1752	found = true;
1753	break;
1754	default:
1755	break;
1756	}
1757	}
1758	if (!found) { /* We don't care. */
1759	netif_dbg(efx, drv, efx->net_dev,
1760	"Ignoring foreign filter that doesn't egdev us\n");
1761	rc = -EOPNOTSUPP;
1762	goto release;
1763	}
1764
1765	rc = efx_mae_match_check_caps(efx, mask: &match.mask, extack);
1766	if (rc)
1767	goto release;
1768
1769	if (efx_tc_match_is_encap(mask: &match.mask)) {
1770	enum efx_encap_type type;
1771
1772	type = efx_tc_indr_netdev_type(net_dev);
1773	if (type == EFX_ENCAP_TYPE_NONE) {
1774	NL_SET_ERR_MSG_MOD(extack,
1775	"Egress encap match on unsupported tunnel device");
1776	rc = -EOPNOTSUPP;
1777	goto release;
1778	}
1779
1780	rc = efx_mae_check_encap_type_supported(efx, typ: type);
1781	if (rc) {
1782	NL_SET_ERR_MSG_FMT_MOD(extack,
1783	"Firmware reports no support for %s encap match",
1784	efx_tc_encap_type_name(type));
1785	goto release;
1786	}
1787
1788	rc = efx_tc_flower_record_encap_match(efx, match: &match, type,
1789	em_type: EFX_TC_EM_DIRECT, child_ip_tos_mask: 0, child_udp_sport_mask: 0,
1790	extack);
1791	if (rc)
1792	goto release;
1793	} else if (!tc->common.chain_index) {
1794	/* This is not a tunnel decap rule, ignore it */
1795	netif_dbg(efx, drv, efx->net_dev,
1796	"Ignoring foreign filter without encap match\n");
1797	rc = -EOPNOTSUPP;
1798	goto release;
1799	}
1800
1801	rule = kzalloc(size: sizeof(*rule), GFP_USER);
1802	if (!rule) {
1803	rc = -ENOMEM;
1804	goto release;
1805	}
1806	INIT_LIST_HEAD(list: &rule->acts.list);
1807	rule->cookie = tc->cookie;
1808	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->match_action_ht,
1809	obj: &rule->linkage,
1810	params: efx_tc_match_action_ht_params);
1811	if (IS_ERR(ptr: old)) {
1812	rc = PTR_ERR(ptr: old);
1813	goto release;
1814	} else if (old) {
1815	netif_dbg(efx, drv, efx->net_dev,
1816	"Ignoring already-offloaded rule (cookie %lx)\n",
1817	tc->cookie);
1818	rc = -EEXIST;
1819	goto release;
1820	}
1821
1822	act = kzalloc(size: sizeof(*act), GFP_USER);
1823	if (!act) {
1824	rc = -ENOMEM;
1825	goto release;
1826	}
1827
1828	/* Parse actions. For foreign rules we only support decap & redirect.
1829	* See corresponding code in efx_tc_flower_replace() for theory of
1830	* operation & how 'act' cursor is used.
1831	*/
1832	flow_action_for_each(i, fa, &fr->action) {
1833	struct efx_tc_action_set save;
1834
1835	switch (fa->id) {
1836	case FLOW_ACTION_REDIRECT:
1837	case FLOW_ACTION_MIRRED:
1838	/* See corresponding code in efx_tc_flower_replace() for
1839	* long explanations of what's going on here.
1840	*/
1841	save = *act;
1842	if (fa->hw_stats) {
1843	struct efx_tc_counter_index *ctr;
1844
1845	if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1846	NL_SET_ERR_MSG_FMT_MOD(extack,
1847	"hw_stats_type %u not supported (only 'delayed')",
1848	fa->hw_stats);
1849	rc = -EOPNOTSUPP;
1850	goto release;
1851	}
1852	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_COUNT)) {
1853	NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)");
1854	rc = -EOPNOTSUPP;
1855	goto release;
1856	}
1857
1858	ctr = efx_tc_flower_get_counter_index(efx,
1859	cookie: tc->cookie,
1860	type: EFX_TC_COUNTER_TYPE_AR);
1861	if (IS_ERR(ptr: ctr)) {
1862	rc = PTR_ERR(ptr: ctr);
1863	NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1864	goto release;
1865	}
1866	act->count = ctr;
1867	INIT_LIST_HEAD(list: &act->count_user);
1868	}
1869
1870	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_DELIVER)) {
1871	/* can't happen */
1872	rc = -EOPNOTSUPP;
1873	NL_SET_ERR_MSG_MOD(extack,
1874	"Deliver action violates action order (can't happen)");
1875	goto release;
1876	}
1877	to_efv = efx_tc_flower_lookup_efv(efx, dev: fa->dev);
1878	/* PF implies egdev is us, in which case we really
1879	* want to deliver to the uplink (because this is an
1880	* ingress filter). If we don't recognise the egdev
1881	* at all, then we'd better trap so SW can handle it.
1882	*/
1883	if (IS_ERR(ptr: to_efv))
1884	to_efv = EFX_EFV_PF;
1885	if (to_efv == EFX_EFV_PF) {
1886	if (uplinked)
1887	break;
1888	uplinked = true;
1889	}
1890	rc = efx_tc_flower_internal_mport(efx, efv: to_efv);
1891	if (rc < 0) {
1892	NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
1893	goto release;
1894	}
1895	act->dest_mport = rc;
1896	act->deliver = 1;
1897	rc = efx_mae_alloc_action_set(efx, act);
1898	if (rc) {
1899	NL_SET_ERR_MSG_MOD(extack,
1900	"Failed to write action set to hw (mirred)");
1901	goto release;
1902	}
1903	list_add_tail(new: &act->list, head: &rule->acts.list);
1904	act = NULL;
1905	if (fa->id == FLOW_ACTION_REDIRECT)
1906	break; /* end of the line */
1907	/* Mirror, so continue on with saved act */
1908	act = kzalloc(size: sizeof(*act), GFP_USER);
1909	if (!act) {
1910	rc = -ENOMEM;
1911	goto release;
1912	}
1913	*act = save;
1914	break;
1915	case FLOW_ACTION_TUNNEL_DECAP:
1916	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_DECAP)) {
1917	rc = -EINVAL;
1918	NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");
1919	goto release;
1920	}
1921	act->decap = 1;
1922	/* If we previously delivered/trapped to uplink, now
1923	* that we've decapped we'll want another copy if we
1924	* try to deliver/trap to uplink again.
1925	*/
1926	uplinked = false;
1927	break;
1928	default:
1929	NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
1930	fa->id);
1931	rc = -EOPNOTSUPP;
1932	goto release;
1933	}
1934	}
1935
1936	if (act) {
1937	if (!uplinked) {
1938	/* Not shot/redirected, so deliver to default dest (which is
1939	* the uplink, as this is an ingress filter)
1940	*/
1941	efx_mae_mport_uplink(efx, out: &act->dest_mport);
1942	act->deliver = 1;
1943	}
1944	rc = efx_mae_alloc_action_set(efx, act);
1945	if (rc) {
1946	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
1947	goto release;
1948	}
1949	list_add_tail(new: &act->list, head: &rule->acts.list);
1950	act = NULL; /* Prevent double-free in error path */
1951	}
1952
1953	rule->match = match;
1954
1955	netif_dbg(efx, drv, efx->net_dev,
1956	"Successfully parsed foreign filter (cookie %lx)\n",
1957	tc->cookie);
1958
1959	rc = efx_mae_alloc_action_set_list(efx, acts: &rule->acts);
1960	if (rc) {
1961	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
1962	goto release;
1963	}
1964	rc = efx_mae_insert_rule(efx, match: &rule->match, prio: EFX_TC_PRIO_TC,
1965	acts_id: rule->acts.fw_id, id: &rule->fw_id);
1966	if (rc) {
1967	NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1968	goto release_acts;
1969	}
1970	return 0;
1971
1972	release_acts:
1973	efx_mae_free_action_set_list(efx, acts: &rule->acts);
1974	release:
1975	/* We failed to insert the rule, so free up any entries we created in
1976	* subsidiary tables.
1977	*/
1978	if (match.rid)
1979	efx_tc_put_recirc_id(efx, rid: match.rid);
1980	if (act)
1981	efx_tc_free_action_set(efx, act, in_hw: false);
1982	if (rule) {
1983	if (!old)
1984	rhashtable_remove_fast(ht: &efx->tc->match_action_ht,
1985	obj: &rule->linkage,
1986	params: efx_tc_match_action_ht_params);
1987	efx_tc_free_action_set_list(efx, acts: &rule->acts, in_hw: false);
1988	}
1989	kfree(objp: rule);
1990	if (match.encap)
1991	efx_tc_flower_release_encap_match(efx, encap: match.encap);
1992	return rc;
1993	}
1994
1995	static int efx_tc_flower_replace_lhs(struct efx_nic *efx,
1996	struct flow_cls_offload *tc,
1997	struct flow_rule *fr,
1998	struct efx_tc_match *match,
1999	struct efx_rep *efv,
2000	struct net_device *net_dev)
2001	{
2002	struct netlink_ext_ack *extack = tc->common.extack;
2003	struct efx_tc_lhs_rule *rule, *old;
2004	int rc;
2005
2006	if (tc->common.chain_index) {
2007	NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
2008	return -EOPNOTSUPP;
2009	}
2010
2011	if (match->mask.ct_state_trk && match->value.ct_state_trk) {
2012	NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
2013	return -EOPNOTSUPP;
2014	}
2015	/* LHS rules are always -trk, so we don't need to match on that */
2016	match->mask.ct_state_trk = 0;
2017	match->value.ct_state_trk = 0;
2018
2019	rc = efx_mae_match_check_caps_lhs(efx, mask: &match->mask, extack);
2020	if (rc)
2021	return rc;
2022
2023	rule = kzalloc(size: sizeof(*rule), GFP_USER);
2024	if (!rule)
2025	return -ENOMEM;
2026	rule->cookie = tc->cookie;
2027	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->lhs_rule_ht,
2028	obj: &rule->linkage,
2029	params: efx_tc_lhs_rule_ht_params);
2030	if (IS_ERR(ptr: old)) {
2031	rc = PTR_ERR(ptr: old);
2032	goto release;
2033	} else if (old) {
2034	netif_dbg(efx, drv, efx->net_dev,
2035	"Already offloaded rule (cookie %lx)\n", tc->cookie);
2036	rc = -EEXIST;
2037	NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2038	goto release;
2039	}
2040
2041	/* Parse actions */
2042	/* See note in efx_tc_flower_replace() regarding passed net_dev
2043	* (used for efx_tc_get_recirc_id()).
2044	*/
2045	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev: efx->net_dev, rule);
2046	if (rc)
2047	goto release;
2048
2049	rule->match = *match;
2050
2051	rc = efx_mae_insert_lhs_rule(efx, rule, prio: EFX_TC_PRIO_TC);
2052	if (rc) {
2053	NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2054	goto release;
2055	}
2056	netif_dbg(efx, drv, efx->net_dev,
2057	"Successfully parsed lhs rule (cookie %lx)\n",
2058	tc->cookie);
2059	return 0;
2060
2061	release:
2062	efx_tc_flower_release_lhs_actions(efx, act: &rule->lhs_act);
2063	if (!old)
2064	rhashtable_remove_fast(ht: &efx->tc->lhs_rule_ht, obj: &rule->linkage,
2065	params: efx_tc_lhs_rule_ht_params);
2066	kfree(objp: rule);
2067	return rc;
2068	}
2069
2070	static int efx_tc_flower_replace(struct efx_nic *efx,
2071	struct net_device *net_dev,
2072	struct flow_cls_offload *tc,
2073	struct efx_rep *efv)
2074	{
2075	struct flow_rule *fr = flow_cls_offload_flow_rule(flow_cmd: tc);
2076	struct netlink_ext_ack *extack = tc->common.extack;
2077	const struct ip_tunnel_info *encap_info = NULL;
2078	struct efx_tc_flow_rule *rule = NULL, *old;
2079	struct efx_tc_mangler_state mung = {};
2080	struct efx_tc_action_set *act = NULL;
2081	const struct flow_action_entry *fa;
2082	struct efx_rep *from_efv, *to_efv;
2083	struct efx_tc_match match;
2084	u32 acts_id;
2085	s64 rc;
2086	int i;
2087
2088	if (!tc_can_offload_extack(dev: efx->net_dev, extack))
2089	return -EOPNOTSUPP;
2090	if (WARN_ON(!efx->tc))
2091	return -ENETDOWN;
2092	if (WARN_ON(!efx->tc->up))
2093	return -ENETDOWN;
2094
2095	from_efv = efx_tc_flower_lookup_efv(efx, dev: net_dev);
2096	if (IS_ERR(ptr: from_efv)) {
2097	/* Not from our PF or representors, so probably a tunnel dev */
2098	return efx_tc_flower_replace_foreign(efx, net_dev, tc);
2099	}
2100
2101	if (efv != from_efv) {
2102	/* can't happen */
2103	NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",
2104	netdev_name(net_dev), efv ? "non-" : "",
2105	from_efv ? "non-" : "");
2106	return -EINVAL;
2107	}
2108
2109	/* Parse match */
2110	memset(&match, 0, sizeof(match));
2111	rc = efx_tc_flower_external_mport(efx, efv: from_efv);
2112	if (rc < 0) {
2113	NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");
2114	return rc;
2115	}
2116	match.value.ingress_port = rc;
2117	match.mask.ingress_port = ~0;
2118	rc = efx_tc_flower_parse_match(efx, rule: fr, match: &match, extack);
2119	if (rc)
2120	return rc;
2121	if (efx_tc_match_is_encap(mask: &match.mask)) {
2122	NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");
2123	return -EOPNOTSUPP;
2124	}
2125
2126	if (efx_tc_rule_is_lhs_rule(fr, match: &match))
2127	return efx_tc_flower_replace_lhs(efx, tc, fr, match: &match, efv,
2128	net_dev);
2129
2130	/* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht).
2131	* Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing
2132	* to the initial memset(), so we don't need to do anything in that case.
2133	*/
2134	if (tc->common.chain_index) {
2135	struct efx_tc_recirc_id *rid;
2136
2137	/* Note regarding passed net_dev:
2138	* VFreps and PF can share chain namespace, as they have
2139	* distinct ingress_mports. So we don't need to burn an
2140	* extra recirc_id if both use the same chain_index.
2141	* (Strictly speaking, we could give each VFrep its own
2142	* recirc_id namespace that doesn't take IDs away from the
2143	* PF, but that would require a bunch of additional IDAs -
2144	* one for each representor - and that's not likely to be
2145	* the main cause of recirc_id exhaustion anyway.)
2146	*/
2147	rid = efx_tc_get_recirc_id(efx, chain_index: tc->common.chain_index,
2148	net_dev: efx->net_dev);
2149	if (IS_ERR(ptr: rid)) {
2150	NL_SET_ERR_MSG_FMT_MOD(extack,
2151	"Failed to allocate a hardware recirculation ID for chain_index %u",
2152	tc->common.chain_index);
2153	return PTR_ERR(ptr: rid);
2154	}
2155	match.rid = rid;
2156	match.value.recirc_id = rid->fw_id;
2157	}
2158	match.mask.recirc_id = 0xff;
2159
2160	/* AR table can't match on DO_CT (+trk). But a commonly used pattern is
2161	* +trk+est, which is strictly implied by +est, so rewrite it to that.
2162	*/
2163	if (match.mask.ct_state_trk && match.value.ct_state_trk &&
2164	match.mask.ct_state_est && match.value.ct_state_est)
2165	match.mask.ct_state_trk = 0;
2166	/* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
2167	* match +trk-est (CT_HIT=0) despite being on an established connection.
2168	* So make -est imply -tcp_syn_fin_rst match to ensure these packets
2169	* still hit the software path.
2170	*/
2171	if (match.mask.ct_state_est && !match.value.ct_state_est) {
2172	if (match.value.tcp_syn_fin_rst) {
2173	/* Can't offload this combination */
2174	rc = -EOPNOTSUPP;
2175	goto release;
2176	}
2177	match.mask.tcp_syn_fin_rst = true;
2178	}
2179
2180	rc = efx_mae_match_check_caps(efx, mask: &match.mask, extack);
2181	if (rc)
2182	goto release;
2183
2184	rule = kzalloc(size: sizeof(*rule), GFP_USER);
2185	if (!rule) {
2186	rc = -ENOMEM;
2187	goto release;
2188	}
2189	INIT_LIST_HEAD(list: &rule->acts.list);
2190	rule->cookie = tc->cookie;
2191	old = rhashtable_lookup_get_insert_fast(ht: &efx->tc->match_action_ht,
2192	obj: &rule->linkage,
2193	params: efx_tc_match_action_ht_params);
2194	if (IS_ERR(ptr: old)) {
2195	rc = PTR_ERR(ptr: old);
2196	goto release;
2197	} else if (old) {
2198	netif_dbg(efx, drv, efx->net_dev,
2199	"Already offloaded rule (cookie %lx)\n", tc->cookie);
2200	NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2201	rc = -EEXIST;
2202	goto release;
2203	}
2204
2205	/* Parse actions */
2206	act = kzalloc(size: sizeof(*act), GFP_USER);
2207	if (!act) {
2208	rc = -ENOMEM;
2209	goto release;
2210	}
2211
2212	/**
2213	* DOC: TC action translation
2214	*
2215	* Actions in TC are sequential and cumulative, with delivery actions
2216	* potentially anywhere in the order. The EF100 MAE, however, takes
2217	* an 'action set list' consisting of 'action sets', each of which is
2218	* applied to the _original_ packet, and consists of a set of optional
2219	* actions in a fixed order with delivery at the end.
2220	* To translate between these two models, we maintain a 'cursor', @act,
2221	* which describes the cumulative effect of all the packet-mutating
2222	* actions encountered so far; on handling a delivery (mirred or drop)
2223	* action, once the action-set has been inserted into hardware, we
2224	* append @act to the action-set list (@rule->acts); if this is a pipe
2225	* action (mirred mirror) we then allocate a new @act with a copy of
2226	* the cursor state _before_ the delivery action, otherwise we set @act
2227	* to %NULL.
2228	* This ensures that every allocated action-set is either attached to
2229	* @rule->acts or pointed to by @act (and never both), and that only
2230	* those action-sets in @rule->acts exist in hardware. Consequently,
2231	* in the failure path, @act only needs to be freed in memory, whereas
2232	* for @rule->acts we remove each action-set from hardware before
2233	* freeing it (efx_tc_free_action_set_list()), even if the action-set
2234	* list itself is not in hardware.
2235	*/
2236	flow_action_for_each(i, fa, &fr->action) {
2237	struct efx_tc_action_set save;
2238	u16 tci;
2239
2240	if (!act) {
2241	/* more actions after a non-pipe action */
2242	NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
2243	rc = -EINVAL;
2244	goto release;
2245	}
2246
2247	if ((fa->id == FLOW_ACTION_REDIRECT ||
2248	fa->id == FLOW_ACTION_MIRRED ||
2249	fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
2250	struct efx_tc_counter_index *ctr;
2251
2252	/* Currently the only actions that want stats are
2253	* mirred and gact (ok, shot, trap, goto-chain), which
2254	* means we want stats just before delivery. Also,
2255	* note that tunnel_key set shouldn't change the length
2256	* — it's only the subsequent mirred that does that,
2257	* and the stats are taken _before_ the mirred action
2258	* happens.
2259	*/
2260	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_COUNT)) {
2261	/* All supported actions that count either steal
2262	* (gact shot, mirred redirect) or clone act
2263	* (mirred mirror), so we should never get two
2264	* count actions on one action_set.
2265	*/
2266	NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
2267	rc = -EOPNOTSUPP;
2268	goto release;
2269	}
2270
2271	if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
2272	NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
2273	fa->hw_stats);
2274	rc = -EOPNOTSUPP;
2275	goto release;
2276	}
2277
2278	ctr = efx_tc_flower_get_counter_index(efx, cookie: tc->cookie,
2279	type: EFX_TC_COUNTER_TYPE_AR);
2280	if (IS_ERR(ptr: ctr)) {
2281	rc = PTR_ERR(ptr: ctr);
2282	NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
2283	goto release;
2284	}
2285	act->count = ctr;
2286	INIT_LIST_HEAD(list: &act->count_user);
2287	}
2288
2289	switch (fa->id) {
2290	case FLOW_ACTION_DROP:
2291	rc = efx_mae_alloc_action_set(efx, act);
2292	if (rc) {
2293	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");
2294	goto release;
2295	}
2296	list_add_tail(new: &act->list, head: &rule->acts.list);
2297	act = NULL; /* end of the line */
2298	break;
2299	case FLOW_ACTION_REDIRECT:
2300	case FLOW_ACTION_MIRRED:
2301	save = *act;
2302
2303	if (encap_info) {
2304	struct efx_tc_encap_action *encap;
2305
2306	if (!efx_tc_flower_action_order_ok(act,
2307	new: EFX_TC_AO_ENCAP)) {
2308	rc = -EOPNOTSUPP;
2309	NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order");
2310	goto release;
2311	}
2312	encap = efx_tc_flower_create_encap_md(
2313	efx, info: encap_info, egdev: fa->dev, extack);
2314	if (IS_ERR_OR_NULL(ptr: encap)) {
2315	rc = PTR_ERR(ptr: encap);
2316	if (!rc)
2317	rc = -EIO; /* arbitrary */
2318	goto release;
2319	}
2320	act->encap_md = encap;
2321	list_add_tail(new: &act->encap_user, head: &encap->users);
2322	act->dest_mport = encap->dest_mport;
2323	act->deliver = 1;
2324	if (act->count && !WARN_ON(!act->count->cnt)) {
2325	/* This counter is used by an encap
2326	* action, which needs a reference back
2327	* so it can prod neighbouring whenever
2328	* traffic is seen.
2329	*/
2330	spin_lock_bh(lock: &act->count->cnt->lock);
2331	list_add_tail(new: &act->count_user,
2332	head: &act->count->cnt->users);
2333	spin_unlock_bh(lock: &act->count->cnt->lock);
2334	}
2335	rc = efx_mae_alloc_action_set(efx, act);
2336	if (rc) {
2337	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)");
2338	goto release;
2339	}
2340	list_add_tail(new: &act->list, head: &rule->acts.list);
2341	act->user = &rule->acts;
2342	act = NULL;
2343	if (fa->id == FLOW_ACTION_REDIRECT)
2344	break; /* end of the line */
2345	/* Mirror, so continue on with saved act */
2346	save.count = NULL;
2347	act = kzalloc(size: sizeof(*act), GFP_USER);
2348	if (!act) {
2349	rc = -ENOMEM;
2350	goto release;
2351	}
2352	*act = save;
2353	break;
2354	}
2355
2356	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_DELIVER)) {
2357	/* can't happen */
2358	rc = -EOPNOTSUPP;
2359	NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
2360	goto release;
2361	}
2362
2363	to_efv = efx_tc_flower_lookup_efv(efx, dev: fa->dev);
2364	if (IS_ERR(ptr: to_efv)) {
2365	NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
2366	rc = PTR_ERR(ptr: to_efv);
2367	goto release;
2368	}
2369	rc = efx_tc_flower_external_mport(efx, efv: to_efv);
2370	if (rc < 0) {
2371	NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
2372	goto release;
2373	}
2374	act->dest_mport = rc;
2375	act->deliver = 1;
2376	rc = efx_mae_alloc_action_set(efx, act);
2377	if (rc) {
2378	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");
2379	goto release;
2380	}
2381	list_add_tail(new: &act->list, head: &rule->acts.list);
2382	act = NULL;
2383	if (fa->id == FLOW_ACTION_REDIRECT)
2384	break; /* end of the line */
2385	/* Mirror, so continue on with saved act */
2386	save.count = NULL;
2387	act = kzalloc(size: sizeof(*act), GFP_USER);
2388	if (!act) {
2389	rc = -ENOMEM;
2390	goto release;
2391	}
2392	*act = save;
2393	break;
2394	case FLOW_ACTION_VLAN_POP:
2395	if (act->vlan_push) {
2396	act->vlan_push--;
2397	} else if (efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_VLAN_POP)) {
2398	act->vlan_pop++;
2399	} else {
2400	NL_SET_ERR_MSG_MOD(extack,
2401	"More than two VLAN pops, or action order violated");
2402	rc = -EINVAL;
2403	goto release;
2404	}
2405	break;
2406	case FLOW_ACTION_VLAN_PUSH:
2407	if (!efx_tc_flower_action_order_ok(act, new: EFX_TC_AO_VLAN_PUSH)) {
2408	rc = -EINVAL;
2409	NL_SET_ERR_MSG_MOD(extack,
2410	"More than two VLAN pushes, or action order violated");
2411	goto release;
2412	}
2413	tci = fa->vlan.vid & VLAN_VID_MASK;
2414	tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;
2415	act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);
2416	act->vlan_proto[act->vlan_push] = fa->vlan.proto;
2417	act->vlan_push++;
2418	break;
2419	case FLOW_ACTION_ADD:
2420	rc = efx_tc_pedit_add(efx, act, fa, extack);
2421	if (rc < 0)
2422	goto release;
2423	break;
2424	case FLOW_ACTION_MANGLE:
2425	rc = efx_tc_mangle(efx, act, fa, mung: &mung, extack, match: &match);
2426	if (rc < 0)
2427	goto release;
2428	break;
2429	case FLOW_ACTION_TUNNEL_ENCAP:
2430	if (encap_info) {
2431	/* Can't specify encap multiple times.
2432	* If you want to overwrite an existing
2433	* encap_info, use an intervening
2434	* FLOW_ACTION_TUNNEL_DECAP to clear it.
2435	*/
2436	NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set");
2437	rc = -EINVAL;
2438	goto release;
2439	}
2440	if (!fa->tunnel) {
2441	NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key");
2442	rc = -EOPNOTSUPP;
2443	goto release;
2444	}
2445	encap_info = fa->tunnel;
2446	break;
2447	case FLOW_ACTION_TUNNEL_DECAP:
2448	if (encap_info) {
2449	encap_info = NULL;
2450	break;
2451	}
2452	/* Since we don't support enc_key matches on ingress
2453	* (and if we did there'd be no tunnel-device to give
2454	* us a type), we can't offload a decap that's not
2455	* just undoing a previous encap action.
2456	*/
2457	NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device");
2458	rc = -EOPNOTSUPP;
2459	goto release;
2460	case FLOW_ACTION_CT:
2461	if (fa->ct.action != TCA_CT_ACT_NAT) {
2462	rc = -EOPNOTSUPP;
2463	NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.action);
2464	goto release;
2465	}
2466	act->do_nat = 1;
2467	break;
2468	default:
2469	NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
2470	fa->id);
2471	rc = -EOPNOTSUPP;
2472	goto release;
2473	}
2474	}
2475
2476	rc = efx_tc_incomplete_mangle(mung: &mung, extack);
2477	if (rc < 0)
2478	goto release;
2479	if (act) {
2480	/* Not shot/redirected, so deliver to default dest */
2481	if (from_efv == EFX_EFV_PF)
2482	/* Rule applies to traffic from the wire,
2483	* and default dest is thus the PF
2484	*/
2485	efx_mae_mport_uplink(efx, out: &act->dest_mport);
2486	else
2487	/* Representor, so rule applies to traffic from
2488	* representee, and default dest is thus the rep.
2489	* All reps use the same mport for delivery
2490	*/
2491	efx_mae_mport_mport(efx, mport_id: efx->tc->reps_mport_id,
2492	out: &act->dest_mport);
2493	act->deliver = 1;
2494	rc = efx_mae_alloc_action_set(efx, act);
2495	if (rc) {
2496	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
2497	goto release;
2498	}
2499	list_add_tail(new: &act->list, head: &rule->acts.list);
2500	act = NULL; /* Prevent double-free in error path */
2501	}
2502
2503	netif_dbg(efx, drv, efx->net_dev,
2504	"Successfully parsed filter (cookie %lx)\n",
2505	tc->cookie);
2506
2507	rule->match = match;
2508
2509	rc = efx_mae_alloc_action_set_list(efx, acts: &rule->acts);
2510	if (rc) {
2511	NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
2512	goto release;
2513	}
2514	if (from_efv == EFX_EFV_PF)
2515	/* PF netdev, so rule applies to traffic from wire */
2516	rule->fallback = &efx->tc->facts.pf;
2517	else
2518	/* repdev, so rule applies to traffic from representee */
2519	rule->fallback = &efx->tc->facts.reps;
2520	if (!efx_tc_check_ready(efx, rule)) {
2521	netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n");
2522	acts_id = rule->fallback->fw_id;
2523	} else {
2524	netif_dbg(efx, drv, efx->net_dev, "ready for hw\n");
2525	acts_id = rule->acts.fw_id;
2526	}
2527	rc = efx_mae_insert_rule(efx, match: &rule->match, prio: EFX_TC_PRIO_TC,
2528	acts_id, id: &rule->fw_id);
2529	if (rc) {
2530	NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2531	goto release_acts;
2532	}
2533	return 0;
2534
2535	release_acts:
2536	efx_mae_free_action_set_list(efx, acts: &rule->acts);
2537	release:
2538	/* We failed to insert the rule, so free up any entries we created in
2539	* subsidiary tables.
2540	*/
2541	if (match.rid)
2542	efx_tc_put_recirc_id(efx, rid: match.rid);
2543	if (act)
2544	efx_tc_free_action_set(efx, act, in_hw: false);
2545	if (rule) {
2546	if (!old)
2547	rhashtable_remove_fast(ht: &efx->tc->match_action_ht,
2548	obj: &rule->linkage,
2549	params: efx_tc_match_action_ht_params);
2550	efx_tc_free_action_set_list(efx, acts: &rule->acts, in_hw: false);
2551	}
2552	kfree(objp: rule);
2553	return rc;
2554	}
2555
2556	static int efx_tc_flower_destroy(struct efx_nic *efx,
2557	struct net_device *net_dev,
2558	struct flow_cls_offload *tc)
2559	{
2560	struct netlink_ext_ack *extack = tc->common.extack;
2561	struct efx_tc_lhs_rule *lhs_rule;
2562	struct efx_tc_flow_rule *rule;
2563
2564	lhs_rule = rhashtable_lookup_fast(ht: &efx->tc->lhs_rule_ht, key: &tc->cookie,
2565	params: efx_tc_lhs_rule_ht_params);
2566	if (lhs_rule) {
2567	/* Remove it from HW */
2568	efx_mae_remove_lhs_rule(efx, rule: lhs_rule);
2569	/* Delete it from SW */
2570	efx_tc_flower_release_lhs_actions(efx, act: &lhs_rule->lhs_act);
2571	rhashtable_remove_fast(ht: &efx->tc->lhs_rule_ht, obj: &lhs_rule->linkage,
2572	params: efx_tc_lhs_rule_ht_params);
2573	if (lhs_rule->match.encap)
2574	efx_tc_flower_release_encap_match(efx, encap: lhs_rule->match.encap);
2575	netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n",
2576	lhs_rule->cookie);
2577	kfree(objp: lhs_rule);
2578	return 0;
2579	}
2580
2581	rule = rhashtable_lookup_fast(ht: &efx->tc->match_action_ht, key: &tc->cookie,
2582	params: efx_tc_match_action_ht_params);
2583	if (!rule) {
2584	/* Only log a message if we're the ingress device. Otherwise
2585	* it's a foreign filter and we might just not have been
2586	* interested (e.g. we might not have been the egress device
2587	* either).
2588	*/
2589	if (!IS_ERR(ptr: efx_tc_flower_lookup_efv(efx, dev: net_dev)))
2590	netif_warn(efx, drv, efx->net_dev,
2591	"Filter %lx not found to remove\n", tc->cookie);
2592	NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2593	return -ENOENT;
2594	}
2595
2596	/* Remove it from HW */
2597	efx_tc_delete_rule(efx, rule);
2598	/* Delete it from SW */
2599	rhashtable_remove_fast(ht: &efx->tc->match_action_ht, obj: &rule->linkage,
2600	params: efx_tc_match_action_ht_params);
2601	netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);
2602	kfree(objp: rule);
2603	return 0;
2604	}
2605
2606	static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
2607	struct flow_cls_offload *tc)
2608	{
2609	struct netlink_ext_ack *extack = tc->common.extack;
2610	struct efx_tc_counter_index *ctr;
2611	struct efx_tc_counter *cnt;
2612	u64 packets, bytes;
2613
2614	ctr = efx_tc_flower_find_counter_index(efx, cookie: tc->cookie);
2615	if (!ctr) {
2616	/* See comment in efx_tc_flower_destroy() */
2617	if (!IS_ERR(ptr: efx_tc_flower_lookup_efv(efx, dev: net_dev)))
2618	if (net_ratelimit())
2619	netif_warn(efx, drv, efx->net_dev,
2620	"Filter %lx not found for stats\n",
2621	tc->cookie);
2622	NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2623	return -ENOENT;
2624	}
2625	if (WARN_ON(!ctr->cnt)) /* can't happen */
2626	return -EIO;
2627	cnt = ctr->cnt;
2628
2629	spin_lock_bh(lock: &cnt->lock);
2630	/* Report only new pkts/bytes since last time TC asked */
2631	packets = cnt->packets;
2632	bytes = cnt->bytes;
2633	flow_stats_update(flow_stats: &tc->stats, bytes: bytes - cnt->old_bytes,
2634	pkts: packets - cnt->old_packets, drops: 0, lastused: cnt->touched,
2635	used_hw_stats: FLOW_ACTION_HW_STATS_DELAYED);
2636	cnt->old_packets = packets;
2637	cnt->old_bytes = bytes;
2638	spin_unlock_bh(lock: &cnt->lock);
2639	return 0;
2640	}
2641
2642	int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
2643	struct flow_cls_offload *tc, struct efx_rep *efv)
2644	{
2645	int rc;
2646
2647	if (!efx->tc)
2648	return -EOPNOTSUPP;
2649
2650	mutex_lock(&efx->tc->mutex);
2651	switch (tc->command) {
2652	case FLOW_CLS_REPLACE:
2653	rc = efx_tc_flower_replace(efx, net_dev, tc, efv);
2654	break;
2655	case FLOW_CLS_DESTROY:
2656	rc = efx_tc_flower_destroy(efx, net_dev, tc);
2657	break;
2658	case FLOW_CLS_STATS:
2659	rc = efx_tc_flower_stats(efx, net_dev, tc);
2660	break;
2661	default:
2662	rc = -EOPNOTSUPP;
2663	break;
2664	}
2665	mutex_unlock(lock: &efx->tc->mutex);
2666	return rc;
2667	}
2668
2669	static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,
2670	u32 eg_port, struct efx_tc_flow_rule *rule)
2671	{
2672	struct efx_tc_action_set_list *acts = &rule->acts;
2673	struct efx_tc_match *match = &rule->match;
2674	struct efx_tc_action_set *act;
2675	int rc;
2676
2677	match->value.ingress_port = ing_port;
2678	match->mask.ingress_port = ~0;
2679	act = kzalloc(size: sizeof(*act), GFP_KERNEL);
2680	if (!act)
2681	return -ENOMEM;
2682	act->deliver = 1;
2683	act->dest_mport = eg_port;
2684	rc = efx_mae_alloc_action_set(efx, act);
2685	if (rc)
2686	goto fail1;
2687	EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2688	list_add_tail(new: &act->list, head: &acts->list);
2689	rc = efx_mae_alloc_action_set_list(efx, acts);
2690	if (rc)
2691	goto fail2;
2692	rc = efx_mae_insert_rule(efx, match, prio: EFX_TC_PRIO_DFLT,
2693	acts_id: acts->fw_id, id: &rule->fw_id);
2694	if (rc)
2695	goto fail3;
2696	return 0;
2697	fail3:
2698	efx_mae_free_action_set_list(efx, acts);
2699	fail2:
2700	list_del(entry: &act->list);
2701	efx_mae_free_action_set(efx, fw_id: act->fw_id);
2702	fail1:
2703	kfree(objp: act);
2704	return rc;
2705	}
2706
2707	static int efx_tc_configure_default_rule_pf(struct efx_nic *efx)
2708	{
2709	struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;
2710	u32 ing_port, eg_port;
2711
2712	efx_mae_mport_uplink(efx, out: &ing_port);
2713	efx_mae_mport_wire(efx, out: &eg_port);
2714	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2715	}
2716
2717	static int efx_tc_configure_default_rule_wire(struct efx_nic *efx)
2718	{
2719	struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;
2720	u32 ing_port, eg_port;
2721
2722	efx_mae_mport_wire(efx, out: &ing_port);
2723	efx_mae_mport_uplink(efx, out: &eg_port);
2724	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2725	}
2726
2727	int efx_tc_configure_default_rule_rep(struct efx_rep *efv)
2728	{
2729	struct efx_tc_flow_rule *rule = &efv->dflt;
2730	struct efx_nic *efx = efv->parent;
2731	u32 ing_port, eg_port;
2732
2733	efx_mae_mport_mport(efx, mport_id: efv->mport, out: &ing_port);
2734	efx_mae_mport_mport(efx, mport_id: efx->tc->reps_mport_id, out: &eg_port);
2735	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2736	}
2737
2738	void efx_tc_deconfigure_default_rule(struct efx_nic *efx,
2739	struct efx_tc_flow_rule *rule)
2740	{
2741	if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)
2742	efx_tc_delete_rule(efx, rule);
2743	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
2744	}
2745
2746	static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port,
2747	struct efx_tc_action_set_list *acts)
2748	{
2749	struct efx_tc_action_set *act;
2750	int rc;
2751
2752	act = kzalloc(size: sizeof(*act), GFP_KERNEL);
2753	if (!act)
2754	return -ENOMEM;
2755	act->deliver = 1;
2756	act->dest_mport = eg_port;
2757	rc = efx_mae_alloc_action_set(efx, act);
2758	if (rc)
2759	goto fail1;
2760	EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2761	list_add_tail(new: &act->list, head: &acts->list);
2762	rc = efx_mae_alloc_action_set_list(efx, acts);
2763	if (rc)
2764	goto fail2;
2765	return 0;
2766	fail2:
2767	list_del(entry: &act->list);
2768	efx_mae_free_action_set(efx, fw_id: act->fw_id);
2769	fail1:
2770	kfree(objp: act);
2771	return rc;
2772	}
2773
2774	static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx)
2775	{
2776	struct efx_tc_action_set_list *acts = &efx->tc->facts.pf;
2777	u32 eg_port;
2778
2779	efx_mae_mport_uplink(efx, out: &eg_port);
2780	return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2781	}
2782
2783	static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx)
2784	{
2785	struct efx_tc_action_set_list *acts = &efx->tc->facts.reps;
2786	u32 eg_port;
2787
2788	efx_mae_mport_mport(efx, mport_id: efx->tc->reps_mport_id, out: &eg_port);
2789	return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2790	}
2791
2792	static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx,
2793	struct efx_tc_action_set_list *acts)
2794	{
2795	efx_tc_free_action_set_list(efx, acts, in_hw: true);
2796	}
2797
2798	static int efx_tc_configure_rep_mport(struct efx_nic *efx)
2799	{
2800	u32 rep_mport_label;
2801	int rc;
2802
2803	rc = efx_mae_allocate_mport(efx, id: &efx->tc->reps_mport_id, label: &rep_mport_label);
2804	if (rc)
2805	return rc;
2806	pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n",
2807	efx->tc->reps_mport_id, rep_mport_label);
2808	/* Use mport *selector* as vport ID */
2809	efx_mae_mport_mport(efx, mport_id: efx->tc->reps_mport_id,
2810	out: &efx->tc->reps_mport_vport_id);
2811	return 0;
2812	}
2813
2814	static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx)
2815	{
2816	efx_mae_free_mport(efx, id: efx->tc->reps_mport_id);
2817	efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;
2818	}
2819
2820	int efx_tc_insert_rep_filters(struct efx_nic *efx)
2821	{
2822	struct efx_filter_spec promisc, allmulti;
2823	int rc;
2824
2825	if (efx->type->is_vf)
2826	return 0;
2827	if (!efx->tc)
2828	return 0;
2829	efx_filter_init_rx(spec: &promisc, priority: EFX_FILTER_PRI_REQUIRED, flags: 0, rxq_id: 0);
2830	efx_filter_set_uc_def(spec: &promisc);
2831	efx_filter_set_vport_id(spec: &promisc, vport_id: efx->tc->reps_mport_vport_id);
2832	rc = efx_filter_insert_filter(efx, spec: &promisc, replace_equal: false);
2833	if (rc < 0)
2834	return rc;
2835	efx->tc->reps_filter_uc = rc;
2836	efx_filter_init_rx(spec: &allmulti, priority: EFX_FILTER_PRI_REQUIRED, flags: 0, rxq_id: 0);
2837	efx_filter_set_mc_def(spec: &allmulti);
2838	efx_filter_set_vport_id(spec: &allmulti, vport_id: efx->tc->reps_mport_vport_id);
2839	rc = efx_filter_insert_filter(efx, spec: &allmulti, replace_equal: false);
2840	if (rc < 0)
2841	return rc;
2842	efx->tc->reps_filter_mc = rc;
2843	return 0;
2844	}
2845
2846	void efx_tc_remove_rep_filters(struct efx_nic *efx)
2847	{
2848	if (efx->type->is_vf)
2849	return;
2850	if (!efx->tc)
2851	return;
2852	if (efx->tc->reps_filter_mc >= 0)
2853	efx_filter_remove_id_safe(efx, priority: EFX_FILTER_PRI_REQUIRED, filter_id: efx->tc->reps_filter_mc);
2854	efx->tc->reps_filter_mc = -1;
2855	if (efx->tc->reps_filter_uc >= 0)
2856	efx_filter_remove_id_safe(efx, priority: EFX_FILTER_PRI_REQUIRED, filter_id: efx->tc->reps_filter_uc);
2857	efx->tc->reps_filter_uc = -1;
2858	}
2859
2860	int efx_init_tc(struct efx_nic *efx)
2861	{
2862	int rc;
2863
2864	rc = efx_mae_get_caps(efx, caps: efx->tc->caps);
2865	if (rc)
2866	return rc;
2867	if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)
2868	/* Firmware supports some match fields the driver doesn't know
2869	* about. Not fatal, unless any of those fields are required
2870	* (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.
2871	*/
2872	netif_warn(efx, probe, efx->net_dev,
2873	"FW reports additional match fields %u\n",
2874	efx->tc->caps->match_field_count);
2875	if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {
2876	netif_err(efx, probe, efx->net_dev,
2877	"Too few action prios supported (have %u, need %u)\n",
2878	efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);
2879	return -EIO;
2880	}
2881	rc = efx_tc_configure_default_rule_pf(efx);
2882	if (rc)
2883	return rc;
2884	rc = efx_tc_configure_default_rule_wire(efx);
2885	if (rc)
2886	return rc;
2887	rc = efx_tc_configure_rep_mport(efx);
2888	if (rc)
2889	return rc;
2890	rc = efx_tc_configure_fallback_acts_pf(efx);
2891	if (rc)
2892	return rc;
2893	rc = efx_tc_configure_fallback_acts_reps(efx);
2894	if (rc)
2895	return rc;
2896	rc = efx_mae_get_tables(efx);
2897	if (rc)
2898	return rc;
2899	rc = flow_indr_dev_register(cb: efx_tc_indr_setup_cb, cb_priv: efx);
2900	if (rc)
2901	goto out_free;
2902	efx->tc->up = true;
2903	return 0;
2904	out_free:
2905	efx_mae_free_tables(efx);
2906	return rc;
2907	}
2908
2909	void efx_fini_tc(struct efx_nic *efx)
2910	{
2911	/* We can get called even if efx_init_struct_tc() failed */
2912	if (!efx->tc)
2913	return;
2914	if (efx->tc->up)
2915	flow_indr_dev_unregister(cb: efx_tc_indr_setup_cb, cb_priv: efx, release: efx_tc_block_unbind);
2916	efx_tc_deconfigure_rep_mport(efx);
2917	efx_tc_deconfigure_default_rule(efx, rule: &efx->tc->dflt.pf);
2918	efx_tc_deconfigure_default_rule(efx, rule: &efx->tc->dflt.wire);
2919	efx_tc_deconfigure_fallback_acts(efx, acts: &efx->tc->facts.pf);
2920	efx_tc_deconfigure_fallback_acts(efx, acts: &efx->tc->facts.reps);
2921	efx->tc->up = false;
2922	efx_mae_free_tables(efx);
2923	}
2924
2925	/* At teardown time, all TC filter rules (and thus all resources they created)
2926	* should already have been removed. If we find any in our hashtables, make a
2927	* cursory attempt to clean up the software side.
2928	*/
2929	static void efx_tc_encap_match_free(void *ptr, void *__unused)
2930	{
2931	struct efx_tc_encap_match *encap = ptr;
2932
2933	WARN_ON(refcount_read(&encap->ref));
2934	kfree(objp: encap);
2935	}
2936
2937	static void efx_tc_recirc_free(void *ptr, void *arg)
2938	{
2939	struct efx_tc_recirc_id *rid = ptr;
2940	struct efx_nic *efx = arg;
2941
2942	WARN_ON(refcount_read(&rid->ref));
2943	ida_free(&efx->tc->recirc_ida, id: rid->fw_id);
2944	kfree(objp: rid);
2945	}
2946
2947	static void efx_tc_lhs_free(void *ptr, void *arg)
2948	{
2949	struct efx_tc_lhs_rule *rule = ptr;
2950	struct efx_nic *efx = arg;
2951
2952	netif_err(efx, drv, efx->net_dev,
2953	"tc lhs_rule %lx still present at teardown, removing\n",
2954	rule->cookie);
2955
2956	if (rule->lhs_act.zone)
2957	efx_tc_ct_unregister_zone(efx, ct_zone: rule->lhs_act.zone);
2958	if (rule->lhs_act.count)
2959	efx_tc_flower_put_counter_index(efx, ctr: rule->lhs_act.count);
2960	efx_mae_remove_lhs_rule(efx, rule);
2961
2962	kfree(objp: rule);
2963	}
2964
2965	static void efx_tc_mac_free(void *ptr, void *__unused)
2966	{
2967	struct efx_tc_mac_pedit_action *ped = ptr;
2968
2969	WARN_ON(refcount_read(&ped->ref));
2970	kfree(objp: ped);
2971	}
2972
2973	static void efx_tc_flow_free(void *ptr, void *arg)
2974	{
2975	struct efx_tc_flow_rule *rule = ptr;
2976	struct efx_nic *efx = arg;
2977
2978	netif_err(efx, drv, efx->net_dev,
2979	"tc rule %lx still present at teardown, removing\n",
2980	rule->cookie);
2981
2982	/* Also releases entries in subsidiary tables */
2983	efx_tc_delete_rule(efx, rule);
2984
2985	kfree(objp: rule);
2986	}
2987
2988	int efx_init_struct_tc(struct efx_nic *efx)
2989	{
2990	int rc;
2991
2992	if (efx->type->is_vf)
2993	return 0;
2994
2995	efx->tc = kzalloc(size: sizeof(*efx->tc), GFP_KERNEL);
2996	if (!efx->tc)
2997	return -ENOMEM;
2998	efx->tc->caps = kzalloc(size: sizeof(struct mae_caps), GFP_KERNEL);
2999	if (!efx->tc->caps) {
3000	rc = -ENOMEM;
3001	goto fail_alloc_caps;
3002	}
3003	INIT_LIST_HEAD(list: &efx->tc->block_list);
3004
3005	mutex_init(&efx->tc->mutex);
3006	init_waitqueue_head(&efx->tc->flush_wq);
3007	rc = efx_tc_init_encap_actions(efx);
3008	if (rc < 0)
3009	goto fail_encap_actions;
3010	rc = efx_tc_init_counters(efx);
3011	if (rc < 0)
3012	goto fail_counters;
3013	rc = rhashtable_init(ht: &efx->tc->mac_ht, params: &efx_tc_mac_ht_params);
3014	if (rc < 0)
3015	goto fail_mac_ht;
3016	rc = rhashtable_init(ht: &efx->tc->encap_match_ht, params: &efx_tc_encap_match_ht_params);
3017	if (rc < 0)
3018	goto fail_encap_match_ht;
3019	rc = rhashtable_init(ht: &efx->tc->match_action_ht, params: &efx_tc_match_action_ht_params);
3020	if (rc < 0)
3021	goto fail_match_action_ht;
3022	rc = rhashtable_init(ht: &efx->tc->lhs_rule_ht, params: &efx_tc_lhs_rule_ht_params);
3023	if (rc < 0)
3024	goto fail_lhs_rule_ht;
3025	rc = efx_tc_init_conntrack(efx);
3026	if (rc < 0)
3027	goto fail_conntrack;
3028	rc = rhashtable_init(ht: &efx->tc->recirc_ht, params: &efx_tc_recirc_ht_params);
3029	if (rc < 0)
3030	goto fail_recirc_ht;
3031	ida_init(ida: &efx->tc->recirc_ida);
3032	efx->tc->reps_filter_uc = -1;
3033	efx->tc->reps_filter_mc = -1;
3034	INIT_LIST_HEAD(list: &efx->tc->dflt.pf.acts.list);
3035	efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3036	INIT_LIST_HEAD(list: &efx->tc->dflt.wire.acts.list);
3037	efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3038	INIT_LIST_HEAD(list: &efx->tc->facts.pf.list);
3039	efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3040	INIT_LIST_HEAD(list: &efx->tc->facts.reps.list);
3041	efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3042	efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
3043	return 0;
3044	fail_recirc_ht:
3045	efx_tc_destroy_conntrack(efx);
3046	fail_conntrack:
3047	rhashtable_destroy(ht: &efx->tc->lhs_rule_ht);
3048	fail_lhs_rule_ht:
3049	rhashtable_destroy(ht: &efx->tc->match_action_ht);
3050	fail_match_action_ht:
3051	rhashtable_destroy(ht: &efx->tc->encap_match_ht);
3052	fail_encap_match_ht:
3053	rhashtable_destroy(ht: &efx->tc->mac_ht);
3054	fail_mac_ht:
3055	efx_tc_destroy_counters(efx);
3056	fail_counters:
3057	efx_tc_destroy_encap_actions(efx);
3058	fail_encap_actions:
3059	mutex_destroy(lock: &efx->tc->mutex);
3060	kfree(objp: efx->tc->caps);
3061	fail_alloc_caps:
3062	kfree(objp: efx->tc);
3063	efx->tc = NULL;
3064	return rc;
3065	}
3066
3067	void efx_fini_struct_tc(struct efx_nic *efx)
3068	{
3069	if (!efx->tc)
3070	return;
3071
3072	mutex_lock(&efx->tc->mutex);
3073	EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=
3074	MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3075	EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=
3076	MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3077	EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id !=
3078	MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3079	EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id !=
3080	MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3081	rhashtable_free_and_destroy(ht: &efx->tc->lhs_rule_ht, free_fn: efx_tc_lhs_free, arg: efx);
3082	rhashtable_free_and_destroy(ht: &efx->tc->match_action_ht, free_fn: efx_tc_flow_free,
3083	arg: efx);
3084	rhashtable_free_and_destroy(ht: &efx->tc->encap_match_ht,
3085	free_fn: efx_tc_encap_match_free, NULL);
3086	efx_tc_fini_conntrack(efx);
3087	rhashtable_free_and_destroy(ht: &efx->tc->recirc_ht, free_fn: efx_tc_recirc_free, arg: efx);
3088	WARN_ON(!ida_is_empty(&efx->tc->recirc_ida));
3089	ida_destroy(ida: &efx->tc->recirc_ida);
3090	rhashtable_free_and_destroy(ht: &efx->tc->mac_ht, free_fn: efx_tc_mac_free, NULL);
3091	efx_tc_fini_counters(efx);
3092	efx_tc_fini_encap_actions(efx);
3093	mutex_unlock(lock: &efx->tc->mutex);
3094	mutex_destroy(lock: &efx->tc->mutex);
3095	kfree(objp: efx->tc->caps);
3096	kfree(objp: efx->tc);
3097	efx->tc = NULL;
3098	}
3099