1	// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2	/* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
3
4	#include <linux/kernel.h>
5	#include <linux/types.h>
6	#include <linux/rhashtable.h>
7	#include <linux/bitops.h>
8	#include <linux/in6.h>
9	#include <linux/notifier.h>
10	#include <linux/inetdevice.h>
11	#include <linux/netdevice.h>
12	#include <linux/if_bridge.h>
13	#include <linux/socket.h>
14	#include <linux/route.h>
15	#include <linux/gcd.h>
16	#include <linux/if_macvlan.h>
17	#include <linux/refcount.h>
18	#include <linux/jhash.h>
19	#include <linux/net_namespace.h>
20	#include <linux/mutex.h>
21	#include <linux/genalloc.h>
22	#include <net/netevent.h>
23	#include <net/neighbour.h>
24	#include <net/arp.h>
25	#include <net/inet_dscp.h>
26	#include <net/ip_fib.h>
27	#include <net/ip6_fib.h>
28	#include <net/nexthop.h>
29	#include <net/fib_rules.h>
30	#include <net/ip_tunnels.h>
31	#include <net/l3mdev.h>
32	#include <net/addrconf.h>
33	#include <net/ndisc.h>
34	#include <net/ipv6.h>
35	#include <net/fib_notifier.h>
36	#include <net/switchdev.h>
37
38	#include "spectrum.h"
39	#include "core.h"
40	#include "reg.h"
41	#include "spectrum_cnt.h"
42	#include "spectrum_dpipe.h"
43	#include "spectrum_ipip.h"
44	#include "spectrum_mr.h"
45	#include "spectrum_mr_tcam.h"
46	#include "spectrum_router.h"
47	#include "spectrum_span.h"
48
49	struct mlxsw_sp_fib;
50	struct mlxsw_sp_vr;
51	struct mlxsw_sp_lpm_tree;
52	struct mlxsw_sp_rif_ops;
53
54	struct mlxsw_sp_crif_key {
55	struct net_device *dev;
56	};
57
58	struct mlxsw_sp_crif {
59	struct mlxsw_sp_crif_key key;
60	struct rhash_head ht_node;
61	bool can_destroy;
62	struct list_head nexthop_list;
63	struct mlxsw_sp_rif *rif;
64	};
65
66	static const struct rhashtable_params mlxsw_sp_crif_ht_params = {
67	.key_offset = offsetof(struct mlxsw_sp_crif, key),
68	.key_len = sizeof_field(struct mlxsw_sp_crif, key),
69	.head_offset = offsetof(struct mlxsw_sp_crif, ht_node),
70	};
71
72	struct mlxsw_sp_rif {
73	struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */
74	netdevice_tracker dev_tracker;
75	struct list_head neigh_list;
76	struct mlxsw_sp_fid *fid;
77	unsigned char addr[ETH_ALEN];
78	int mtu;
79	u16 rif_index;
80	u8 mac_profile_id;
81	u8 rif_entries;
82	u16 vr_id;
83	const struct mlxsw_sp_rif_ops *ops;
84	struct mlxsw_sp *mlxsw_sp;
85
86	unsigned int counter_ingress;
87	bool counter_ingress_valid;
88	unsigned int counter_egress;
89	bool counter_egress_valid;
90	};
91
92	static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
93	{
94	if (!rif->crif)
95	return NULL;
96	return rif->crif->key.dev;
97	}
98
99	struct mlxsw_sp_rif_params {
100	struct net_device *dev;
101	union {
102	u16 system_port;
103	u16 lag_id;
104	};
105	u16 vid;
106	bool lag;
107	bool double_entry;
108	};
109
110	struct mlxsw_sp_rif_subport {
111	struct mlxsw_sp_rif common;
112	refcount_t ref_count;
113	union {
114	u16 system_port;
115	u16 lag_id;
116	};
117	u16 vid;
118	bool lag;
119	};
120
121	struct mlxsw_sp_rif_ipip_lb {
122	struct mlxsw_sp_rif common;
123	struct mlxsw_sp_rif_ipip_lb_config lb_config;
124	u16 ul_vr_id; /* Spectrum-1. */
125	u16 ul_rif_id; /* Spectrum-2+. */
126	};
127
128	struct mlxsw_sp_rif_params_ipip_lb {
129	struct mlxsw_sp_rif_params common;
130	struct mlxsw_sp_rif_ipip_lb_config lb_config;
131	};
132
133	struct mlxsw_sp_rif_ops {
134	enum mlxsw_sp_rif_type type;
135	size_t rif_size;
136
137	void (*setup)(struct mlxsw_sp_rif *rif,
138	const struct mlxsw_sp_rif_params *params);
139	int (*configure)(struct mlxsw_sp_rif *rif,
140	struct netlink_ext_ack *extack);
141	void (*deconfigure)(struct mlxsw_sp_rif *rif);
142	struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
143	const struct mlxsw_sp_rif_params *params,
144	struct netlink_ext_ack *extack);
145	void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
146	};
147
148	struct mlxsw_sp_rif_mac_profile {
149	unsigned char mac_prefix[ETH_ALEN];
150	refcount_t ref_count;
151	u8 id;
152	};
153
154	struct mlxsw_sp_router_ops {
155	int (*init)(struct mlxsw_sp *mlxsw_sp);
156	int (*ipips_init)(struct mlxsw_sp *mlxsw_sp);
157	};
158
159	static struct mlxsw_sp_rif *
160	mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
161	const struct net_device *dev);
162	static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
163	static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
164	static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
165	struct mlxsw_sp_lpm_tree *lpm_tree);
166	static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
167	const struct mlxsw_sp_fib *fib,
168	u8 tree_id);
169	static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
170	const struct mlxsw_sp_fib *fib);
171
172	static unsigned int *
173	mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
174	enum mlxsw_sp_rif_counter_dir dir)
175	{
176	switch (dir) {
177	case MLXSW_SP_RIF_COUNTER_EGRESS:
178	return &rif->counter_egress;
179	case MLXSW_SP_RIF_COUNTER_INGRESS:
180	return &rif->counter_ingress;
181	}
182	return NULL;
183	}
184
185	static bool
186	mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
187	enum mlxsw_sp_rif_counter_dir dir)
188	{
189	switch (dir) {
190	case MLXSW_SP_RIF_COUNTER_EGRESS:
191	return rif->counter_egress_valid;
192	case MLXSW_SP_RIF_COUNTER_INGRESS:
193	return rif->counter_ingress_valid;
194	}
195	return false;
196	}
197
198	static void
199	mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
200	enum mlxsw_sp_rif_counter_dir dir,
201	bool valid)
202	{
203	switch (dir) {
204	case MLXSW_SP_RIF_COUNTER_EGRESS:
205	rif->counter_egress_valid = valid;
206	break;
207	case MLXSW_SP_RIF_COUNTER_INGRESS:
208	rif->counter_ingress_valid = valid;
209	break;
210	}
211	}
212
213	static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
214	unsigned int counter_index, bool enable,
215	enum mlxsw_sp_rif_counter_dir dir)
216	{
217	char ritr_pl[MLXSW_REG_RITR_LEN];
218	bool is_egress = false;
219	int err;
220
221	if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
222	is_egress = true;
223	mlxsw_reg_ritr_rif_pack(payload: ritr_pl, rif: rif_index);
224	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
225	if (err)
226	return err;
227
228	mlxsw_reg_ritr_counter_pack(payload: ritr_pl, index: counter_index, enable,
229	egress: is_egress);
230	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
231	}
232
233	int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
234	struct mlxsw_sp_rif *rif,
235	enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
236	{
237	char ricnt_pl[MLXSW_REG_RICNT_LEN];
238	unsigned int *p_counter_index;
239	bool valid;
240	int err;
241
242	valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
243	if (!valid)
244	return -EINVAL;
245
246	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
247	if (!p_counter_index)
248	return -EINVAL;
249	mlxsw_reg_ricnt_pack(payload: ricnt_pl, index: *p_counter_index,
250	op: MLXSW_REG_RICNT_OPCODE_NOP);
251	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ricnt), payload: ricnt_pl);
252	if (err)
253	return err;
254	*cnt = mlxsw_reg_ricnt_good_unicast_packets_get(buf: ricnt_pl);
255	return 0;
256	}
257
258	struct mlxsw_sp_rif_counter_set_basic {
259	u64 good_unicast_packets;
260	u64 good_multicast_packets;
261	u64 good_broadcast_packets;
262	u64 good_unicast_bytes;
263	u64 good_multicast_bytes;
264	u64 good_broadcast_bytes;
265	u64 error_packets;
266	u64 discard_packets;
267	u64 error_bytes;
268	u64 discard_bytes;
269	};
270
271	static int
272	mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif,
273	enum mlxsw_sp_rif_counter_dir dir,
274	struct mlxsw_sp_rif_counter_set_basic *set)
275	{
276	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
277	char ricnt_pl[MLXSW_REG_RICNT_LEN];
278	unsigned int *p_counter_index;
279	int err;
280
281	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
282	return -EINVAL;
283
284	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
285	if (!p_counter_index)
286	return -EINVAL;
287
288	mlxsw_reg_ricnt_pack(payload: ricnt_pl, index: *p_counter_index,
289	op: MLXSW_REG_RICNT_OPCODE_CLEAR);
290	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ricnt), payload: ricnt_pl);
291	if (err)
292	return err;
293
294	if (!set)
295	return 0;
296
297	#define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \
298	(set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl))
299
300	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets);
301	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets);
302	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets);
303	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes);
304	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes);
305	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes);
306	MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets);
307	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets);
308	MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes);
309	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes);
310
311	#undef MLXSW_SP_RIF_COUNTER_EXTRACT
312
313	return 0;
314	}
315
316	static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
317	unsigned int counter_index)
318	{
319	char ricnt_pl[MLXSW_REG_RICNT_LEN];
320
321	mlxsw_reg_ricnt_pack(payload: ricnt_pl, index: counter_index,
322	op: MLXSW_REG_RICNT_OPCODE_CLEAR);
323	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ricnt), payload: ricnt_pl);
324	}
325
326	int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif,
327	enum mlxsw_sp_rif_counter_dir dir)
328	{
329	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
330	unsigned int *p_counter_index;
331	int err;
332
333	if (mlxsw_sp_rif_counter_valid_get(rif, dir))
334	return 0;
335
336	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
337	if (!p_counter_index)
338	return -EINVAL;
339
340	err = mlxsw_sp_counter_alloc(mlxsw_sp, sub_pool_id: MLXSW_SP_COUNTER_SUB_POOL_RIF,
341	p_counter_index);
342	if (err)
343	return err;
344
345	err = mlxsw_sp_rif_counter_clear(mlxsw_sp, counter_index: *p_counter_index);
346	if (err)
347	goto err_counter_clear;
348
349	err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif_index: rif->rif_index,
350	counter_index: *p_counter_index, enable: true, dir);
351	if (err)
352	goto err_counter_edit;
353	mlxsw_sp_rif_counter_valid_set(rif, dir, valid: true);
354	return 0;
355
356	err_counter_edit:
357	err_counter_clear:
358	mlxsw_sp_counter_free(mlxsw_sp, sub_pool_id: MLXSW_SP_COUNTER_SUB_POOL_RIF,
359	counter_index: *p_counter_index);
360	return err;
361	}
362
363	void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif,
364	enum mlxsw_sp_rif_counter_dir dir)
365	{
366	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
367	unsigned int *p_counter_index;
368
369	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
370	return;
371
372	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
373	if (WARN_ON(!p_counter_index))
374	return;
375	mlxsw_sp_rif_counter_edit(mlxsw_sp, rif_index: rif->rif_index,
376	counter_index: *p_counter_index, enable: false, dir);
377	mlxsw_sp_counter_free(mlxsw_sp, sub_pool_id: MLXSW_SP_COUNTER_SUB_POOL_RIF,
378	counter_index: *p_counter_index);
379	mlxsw_sp_rif_counter_valid_set(rif, dir, valid: false);
380	}
381
382	static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
383	{
384	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
385	struct devlink *devlink;
386
387	devlink = priv_to_devlink(priv: mlxsw_sp->core);
388	if (!devlink_dpipe_table_counter_enabled(devlink,
389	MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
390	return;
391	mlxsw_sp_rif_counter_alloc(rif, dir: MLXSW_SP_RIF_COUNTER_EGRESS);
392	}
393
394	static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
395	{
396	mlxsw_sp_rif_counter_free(rif, dir: MLXSW_SP_RIF_COUNTER_EGRESS);
397	}
398
399	#define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
400
401	struct mlxsw_sp_prefix_usage {
402	DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
403	};
404
405	#define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
406	for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
407
408	static bool
409	mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
410	struct mlxsw_sp_prefix_usage *prefix_usage2)
411	{
412	return !memcmp(p: prefix_usage1, q: prefix_usage2, size: sizeof(*prefix_usage1));
413	}
414
415	static void
416	mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
417	struct mlxsw_sp_prefix_usage *prefix_usage2)
418	{
419	memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
420	}
421
422	static void
423	mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
424	unsigned char prefix_len)
425	{
426	set_bit(nr: prefix_len, addr: prefix_usage->b);
427	}
428
429	static void
430	mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
431	unsigned char prefix_len)
432	{
433	clear_bit(nr: prefix_len, addr: prefix_usage->b);
434	}
435
436	struct mlxsw_sp_fib_key {
437	unsigned char addr[sizeof(struct in6_addr)];
438	unsigned char prefix_len;
439	};
440
441	enum mlxsw_sp_fib_entry_type {
442	MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
443	MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
444	MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
445	MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
446	MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
447
448	/* This is a special case of local delivery, where a packet should be
449	* decapsulated on reception. Note that there is no corresponding ENCAP,
450	* because that's a type of next hop, not of FIB entry. (There can be
451	* several next hops in a REMOTE entry, and some of them may be
452	* encapsulating entries.)
453	*/
454	MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
455	MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
456	};
457
458	struct mlxsw_sp_nexthop_group_info;
459	struct mlxsw_sp_nexthop_group;
460	struct mlxsw_sp_fib_entry;
461
462	struct mlxsw_sp_fib_node {
463	struct mlxsw_sp_fib_entry *fib_entry;
464	struct list_head list;
465	struct rhash_head ht_node;
466	struct mlxsw_sp_fib *fib;
467	struct mlxsw_sp_fib_key key;
468	};
469
470	struct mlxsw_sp_fib_entry_decap {
471	struct mlxsw_sp_ipip_entry *ipip_entry;
472	u32 tunnel_index;
473	};
474
475	struct mlxsw_sp_fib_entry {
476	struct mlxsw_sp_fib_node *fib_node;
477	enum mlxsw_sp_fib_entry_type type;
478	struct list_head nexthop_group_node;
479	struct mlxsw_sp_nexthop_group *nh_group;
480	struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
481	};
482
483	struct mlxsw_sp_fib4_entry {
484	struct mlxsw_sp_fib_entry common;
485	struct fib_info *fi;
486	u32 tb_id;
487	dscp_t dscp;
488	u8 type;
489	};
490
491	struct mlxsw_sp_fib6_entry {
492	struct mlxsw_sp_fib_entry common;
493	struct list_head rt6_list;
494	unsigned int nrt6;
495	};
496
497	struct mlxsw_sp_rt6 {
498	struct list_head list;
499	struct fib6_info *rt;
500	};
501
502	struct mlxsw_sp_lpm_tree {
503	u8 id; /* tree ID */
504	unsigned int ref_count;
505	enum mlxsw_sp_l3proto proto;
506	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
507	struct mlxsw_sp_prefix_usage prefix_usage;
508	};
509
510	struct mlxsw_sp_fib {
511	struct rhashtable ht;
512	struct list_head node_list;
513	struct mlxsw_sp_vr *vr;
514	struct mlxsw_sp_lpm_tree *lpm_tree;
515	enum mlxsw_sp_l3proto proto;
516	};
517
518	struct mlxsw_sp_vr {
519	u16 id; /* virtual router ID */
520	u32 tb_id; /* kernel fib table id */
521	unsigned int rif_count;
522	struct mlxsw_sp_fib *fib4;
523	struct mlxsw_sp_fib *fib6;
524	struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
525	struct mlxsw_sp_rif *ul_rif;
526	refcount_t ul_rif_refcnt;
527	};
528
529	static const struct rhashtable_params mlxsw_sp_fib_ht_params;
530
531	static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
532	struct mlxsw_sp_vr *vr,
533	enum mlxsw_sp_l3proto proto)
534	{
535	struct mlxsw_sp_lpm_tree *lpm_tree;
536	struct mlxsw_sp_fib *fib;
537	int err;
538
539	lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
540	fib = kzalloc(size: sizeof(*fib), GFP_KERNEL);
541	if (!fib)
542	return ERR_PTR(error: -ENOMEM);
543	err = rhashtable_init(ht: &fib->ht, params: &mlxsw_sp_fib_ht_params);
544	if (err)
545	goto err_rhashtable_init;
546	INIT_LIST_HEAD(list: &fib->node_list);
547	fib->proto = proto;
548	fib->vr = vr;
549	fib->lpm_tree = lpm_tree;
550	mlxsw_sp_lpm_tree_hold(lpm_tree);
551	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, tree_id: lpm_tree->id);
552	if (err)
553	goto err_lpm_tree_bind;
554	return fib;
555
556	err_lpm_tree_bind:
557	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
558	err_rhashtable_init:
559	kfree(objp: fib);
560	return ERR_PTR(error: err);
561	}
562
563	static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
564	struct mlxsw_sp_fib *fib)
565	{
566	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
567	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree: fib->lpm_tree);
568	WARN_ON(!list_empty(&fib->node_list));
569	rhashtable_destroy(ht: &fib->ht);
570	kfree(objp: fib);
571	}
572
573	static struct mlxsw_sp_lpm_tree *
574	mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
575	{
576	static struct mlxsw_sp_lpm_tree *lpm_tree;
577	int i;
578
579	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
580	lpm_tree = &mlxsw_sp->router->lpm.trees[i];
581	if (lpm_tree->ref_count == 0)
582	return lpm_tree;
583	}
584	return NULL;
585	}
586
587	static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
588	struct mlxsw_sp_lpm_tree *lpm_tree)
589	{
590	char ralta_pl[MLXSW_REG_RALTA_LEN];
591
592	mlxsw_reg_ralta_pack(payload: ralta_pl, alloc: true,
593	protocol: (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
594	tree_id: lpm_tree->id);
595	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralta), payload: ralta_pl);
596	}
597
598	static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
599	struct mlxsw_sp_lpm_tree *lpm_tree)
600	{
601	char ralta_pl[MLXSW_REG_RALTA_LEN];
602
603	mlxsw_reg_ralta_pack(payload: ralta_pl, alloc: false,
604	protocol: (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
605	tree_id: lpm_tree->id);
606	mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralta), payload: ralta_pl);
607	}
608
609	static int
610	mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
611	struct mlxsw_sp_prefix_usage *prefix_usage,
612	struct mlxsw_sp_lpm_tree *lpm_tree)
613	{
614	char ralst_pl[MLXSW_REG_RALST_LEN];
615	u8 root_bin = 0;
616	u8 prefix;
617	u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
618
619	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
620	root_bin = prefix;
621
622	mlxsw_reg_ralst_pack(payload: ralst_pl, root_bin, tree_id: lpm_tree->id);
623	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
624	if (prefix == 0)
625	continue;
626	mlxsw_reg_ralst_bin_pack(payload: ralst_pl, bin_number: prefix, left_child_bin: last_prefix,
627	MLXSW_REG_RALST_BIN_NO_CHILD);
628	last_prefix = prefix;
629	}
630	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralst), payload: ralst_pl);
631	}
632
633	static struct mlxsw_sp_lpm_tree *
634	mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
635	struct mlxsw_sp_prefix_usage *prefix_usage,
636	enum mlxsw_sp_l3proto proto)
637	{
638	struct mlxsw_sp_lpm_tree *lpm_tree;
639	int err;
640
641	lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
642	if (!lpm_tree)
643	return ERR_PTR(error: -EBUSY);
644	lpm_tree->proto = proto;
645	err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
646	if (err)
647	return ERR_PTR(error: err);
648
649	err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
650	lpm_tree);
651	if (err)
652	goto err_left_struct_set;
653	memcpy(&lpm_tree->prefix_usage, prefix_usage,
654	sizeof(lpm_tree->prefix_usage));
655	memset(&lpm_tree->prefix_ref_count, 0,
656	sizeof(lpm_tree->prefix_ref_count));
657	lpm_tree->ref_count = 1;
658	return lpm_tree;
659
660	err_left_struct_set:
661	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
662	return ERR_PTR(error: err);
663	}
664
665	static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
666	struct mlxsw_sp_lpm_tree *lpm_tree)
667	{
668	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
669	}
670
671	static struct mlxsw_sp_lpm_tree *
672	mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
673	struct mlxsw_sp_prefix_usage *prefix_usage,
674	enum mlxsw_sp_l3proto proto)
675	{
676	struct mlxsw_sp_lpm_tree *lpm_tree;
677	int i;
678
679	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
680	lpm_tree = &mlxsw_sp->router->lpm.trees[i];
681	if (lpm_tree->ref_count != 0 &&
682	lpm_tree->proto == proto &&
683	mlxsw_sp_prefix_usage_eq(prefix_usage1: &lpm_tree->prefix_usage,
684	prefix_usage2: prefix_usage)) {
685	mlxsw_sp_lpm_tree_hold(lpm_tree);
686	return lpm_tree;
687	}
688	}
689	return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
690	}
691
692	static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
693	{
694	lpm_tree->ref_count++;
695	}
696
697	static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
698	struct mlxsw_sp_lpm_tree *lpm_tree)
699	{
700	if (--lpm_tree->ref_count == 0)
701	mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
702	}
703
704	#define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
705
706	static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
707	{
708	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
709	struct mlxsw_sp_lpm_tree *lpm_tree;
710	u64 max_trees;
711	int err, i;
712
713	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
714	return -EIO;
715
716	max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
717	mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
718	mlxsw_sp->router->lpm.trees = kcalloc(n: mlxsw_sp->router->lpm.tree_count,
719	size: sizeof(struct mlxsw_sp_lpm_tree),
720	GFP_KERNEL);
721	if (!mlxsw_sp->router->lpm.trees)
722	return -ENOMEM;
723
724	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
725	lpm_tree = &mlxsw_sp->router->lpm.trees[i];
726	lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
727	}
728
729	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, prefix_usage: &req_prefix_usage,
730	proto: MLXSW_SP_L3_PROTO_IPV4);
731	if (IS_ERR(ptr: lpm_tree)) {
732	err = PTR_ERR(ptr: lpm_tree);
733	goto err_ipv4_tree_get;
734	}
735	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
736
737	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, prefix_usage: &req_prefix_usage,
738	proto: MLXSW_SP_L3_PROTO_IPV6);
739	if (IS_ERR(ptr: lpm_tree)) {
740	err = PTR_ERR(ptr: lpm_tree);
741	goto err_ipv6_tree_get;
742	}
743	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
744
745	return 0;
746
747	err_ipv6_tree_get:
748	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
749	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
750	err_ipv4_tree_get:
751	kfree(objp: mlxsw_sp->router->lpm.trees);
752	return err;
753	}
754
755	static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
756	{
757	struct mlxsw_sp_lpm_tree *lpm_tree;
758
759	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
760	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
761
762	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
763	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
764
765	kfree(objp: mlxsw_sp->router->lpm.trees);
766	}
767
768	static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
769	{
770	return !!vr->fib4 || !!vr->fib6 ||
771	!!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
772	!!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
773	}
774
775	static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
776	{
777	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
778	struct mlxsw_sp_vr *vr;
779	int i;
780
781	for (i = 0; i < max_vrs; i++) {
782	vr = &mlxsw_sp->router->vrs[i];
783	if (!mlxsw_sp_vr_is_used(vr))
784	return vr;
785	}
786	return NULL;
787	}
788
789	static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
790	const struct mlxsw_sp_fib *fib, u8 tree_id)
791	{
792	char raltb_pl[MLXSW_REG_RALTB_LEN];
793
794	mlxsw_reg_raltb_pack(payload: raltb_pl, virtual_router: fib->vr->id,
795	protocol: (enum mlxsw_reg_ralxx_protocol) fib->proto,
796	tree_id);
797	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(raltb), payload: raltb_pl);
798	}
799
800	static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
801	const struct mlxsw_sp_fib *fib)
802	{
803	char raltb_pl[MLXSW_REG_RALTB_LEN];
804
805	/* Bind to tree 0 which is default */
806	mlxsw_reg_raltb_pack(payload: raltb_pl, virtual_router: fib->vr->id,
807	protocol: (enum mlxsw_reg_ralxx_protocol) fib->proto, tree_id: 0);
808	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(raltb), payload: raltb_pl);
809	}
810
811	static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
812	{
813	/* For our purpose, squash main, default and local tables into one */
814	if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
815	tb_id = RT_TABLE_MAIN;
816	return tb_id;
817	}
818
819	static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
820	u32 tb_id)
821	{
822	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
823	struct mlxsw_sp_vr *vr;
824	int i;
825
826	tb_id = mlxsw_sp_fix_tb_id(tb_id);
827
828	for (i = 0; i < max_vrs; i++) {
829	vr = &mlxsw_sp->router->vrs[i];
830	if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
831	return vr;
832	}
833	return NULL;
834	}
835
836	int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
837	u16 *vr_id)
838	{
839	struct mlxsw_sp_vr *vr;
840	int err = 0;
841
842	mutex_lock(&mlxsw_sp->router->lock);
843	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
844	if (!vr) {
845	err = -ESRCH;
846	goto out;
847	}
848	*vr_id = vr->id;
849	out:
850	mutex_unlock(lock: &mlxsw_sp->router->lock);
851	return err;
852	}
853
854	static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
855	enum mlxsw_sp_l3proto proto)
856	{
857	switch (proto) {
858	case MLXSW_SP_L3_PROTO_IPV4:
859	return vr->fib4;
860	case MLXSW_SP_L3_PROTO_IPV6:
861	return vr->fib6;
862	}
863	return NULL;
864	}
865
866	static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
867	u32 tb_id,
868	struct netlink_ext_ack *extack)
869	{
870	struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
871	struct mlxsw_sp_fib *fib4;
872	struct mlxsw_sp_fib *fib6;
873	struct mlxsw_sp_vr *vr;
874	int err;
875
876	vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
877	if (!vr) {
878	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
879	return ERR_PTR(error: -EBUSY);
880	}
881	fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, proto: MLXSW_SP_L3_PROTO_IPV4);
882	if (IS_ERR(ptr: fib4))
883	return ERR_CAST(ptr: fib4);
884	fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, proto: MLXSW_SP_L3_PROTO_IPV6);
885	if (IS_ERR(ptr: fib6)) {
886	err = PTR_ERR(ptr: fib6);
887	goto err_fib6_create;
888	}
889	mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, tb_id: vr->id,
890	proto: MLXSW_SP_L3_PROTO_IPV4);
891	if (IS_ERR(ptr: mr4_table)) {
892	err = PTR_ERR(ptr: mr4_table);
893	goto err_mr4_table_create;
894	}
895	mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, tb_id: vr->id,
896	proto: MLXSW_SP_L3_PROTO_IPV6);
897	if (IS_ERR(ptr: mr6_table)) {
898	err = PTR_ERR(ptr: mr6_table);
899	goto err_mr6_table_create;
900	}
901
902	vr->fib4 = fib4;
903	vr->fib6 = fib6;
904	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
905	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
906	vr->tb_id = tb_id;
907	return vr;
908
909	err_mr6_table_create:
910	mlxsw_sp_mr_table_destroy(mr_table: mr4_table);
911	err_mr4_table_create:
912	mlxsw_sp_fib_destroy(mlxsw_sp, fib: fib6);
913	err_fib6_create:
914	mlxsw_sp_fib_destroy(mlxsw_sp, fib: fib4);
915	return ERR_PTR(error: err);
916	}
917
918	static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
919	struct mlxsw_sp_vr *vr)
920	{
921	mlxsw_sp_mr_table_destroy(mr_table: vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
922	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
923	mlxsw_sp_mr_table_destroy(mr_table: vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
924	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
925	mlxsw_sp_fib_destroy(mlxsw_sp, fib: vr->fib6);
926	vr->fib6 = NULL;
927	mlxsw_sp_fib_destroy(mlxsw_sp, fib: vr->fib4);
928	vr->fib4 = NULL;
929	}
930
931	static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
932	struct netlink_ext_ack *extack)
933	{
934	struct mlxsw_sp_vr *vr;
935
936	tb_id = mlxsw_sp_fix_tb_id(tb_id);
937	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
938	if (!vr)
939	vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
940	return vr;
941	}
942
943	static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
944	{
945	if (!vr->rif_count && list_empty(head: &vr->fib4->node_list) &&
946	list_empty(head: &vr->fib6->node_list) &&
947	mlxsw_sp_mr_table_empty(mr_table: vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
948	mlxsw_sp_mr_table_empty(mr_table: vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
949	mlxsw_sp_vr_destroy(mlxsw_sp, vr);
950	}
951
952	static bool
953	mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
954	enum mlxsw_sp_l3proto proto, u8 tree_id)
955	{
956	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
957
958	if (!mlxsw_sp_vr_is_used(vr))
959	return false;
960	if (fib->lpm_tree->id == tree_id)
961	return true;
962	return false;
963	}
964
965	static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
966	struct mlxsw_sp_fib *fib,
967	struct mlxsw_sp_lpm_tree *new_tree)
968	{
969	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
970	int err;
971
972	fib->lpm_tree = new_tree;
973	mlxsw_sp_lpm_tree_hold(lpm_tree: new_tree);
974	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, tree_id: new_tree->id);
975	if (err)
976	goto err_tree_bind;
977	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree: old_tree);
978	return 0;
979
980	err_tree_bind:
981	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree: new_tree);
982	fib->lpm_tree = old_tree;
983	return err;
984	}
985
986	static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
987	struct mlxsw_sp_fib *fib,
988	struct mlxsw_sp_lpm_tree *new_tree)
989	{
990	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
991	enum mlxsw_sp_l3proto proto = fib->proto;
992	struct mlxsw_sp_lpm_tree *old_tree;
993	u8 old_id, new_id = new_tree->id;
994	struct mlxsw_sp_vr *vr;
995	int i, err;
996
997	old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
998	old_id = old_tree->id;
999
1000	for (i = 0; i < max_vrs; i++) {
1001	vr = &mlxsw_sp->router->vrs[i];
1002	if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, tree_id: old_id))
1003	continue;
1004	err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1005	fib: mlxsw_sp_vr_fib(vr, proto),
1006	new_tree);
1007	if (err)
1008	goto err_tree_replace;
1009	}
1010
1011	memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
1012	sizeof(new_tree->prefix_ref_count));
1013	mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
1014	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree: old_tree);
1015
1016	return 0;
1017
1018	err_tree_replace:
1019	for (i--; i >= 0; i--) {
1020	if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, tree_id: new_id))
1021	continue;
1022	mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1023	fib: mlxsw_sp_vr_fib(vr, proto),
1024	new_tree: old_tree);
1025	}
1026	return err;
1027	}
1028
1029	static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1030	{
1031	struct mlxsw_sp_vr *vr;
1032	u64 max_vrs;
1033	int i;
1034
1035	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1036	return -EIO;
1037
1038	max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1039	mlxsw_sp->router->vrs = kcalloc(n: max_vrs, size: sizeof(struct mlxsw_sp_vr),
1040	GFP_KERNEL);
1041	if (!mlxsw_sp->router->vrs)
1042	return -ENOMEM;
1043
1044	for (i = 0; i < max_vrs; i++) {
1045	vr = &mlxsw_sp->router->vrs[i];
1046	vr->id = i;
1047	}
1048
1049	return 0;
1050	}
1051
1052	static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1053
1054	static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1055	{
1056	/* At this stage we're guaranteed not to have new incoming
1057	* FIB notifications and the work queue is free from FIBs
1058	* sitting on top of mlxsw netdevs. However, we can still
1059	* have other FIBs queued. Flush the queue before flushing
1060	* the device's tables. No need for locks, as we're the only
1061	* writer.
1062	*/
1063	mlxsw_core_flush_owq();
1064	mlxsw_sp_router_fib_flush(mlxsw_sp);
1065	kfree(objp: mlxsw_sp->router->vrs);
1066	}
1067
1068	u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1069	{
1070	struct net_device *d;
1071	u32 tb_id;
1072
1073	rcu_read_lock();
1074	d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1075	if (d)
1076	tb_id = l3mdev_fib_table(dev: d) ? : RT_TABLE_MAIN;
1077	else
1078	tb_id = RT_TABLE_MAIN;
1079	rcu_read_unlock();
1080
1081	return tb_id;
1082	}
1083
1084	static void
1085	mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev)
1086	{
1087	crif->key.dev = dev;
1088	INIT_LIST_HEAD(list: &crif->nexthop_list);
1089	}
1090
1091	static struct mlxsw_sp_crif *
1092	mlxsw_sp_crif_alloc(struct net_device *dev)
1093	{
1094	struct mlxsw_sp_crif *crif;
1095
1096	crif = kzalloc(size: sizeof(*crif), GFP_KERNEL);
1097	if (!crif)
1098	return NULL;
1099
1100	mlxsw_sp_crif_init(crif, dev);
1101	return crif;
1102	}
1103
1104	static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif)
1105	{
1106	if (WARN_ON(crif->rif))
1107	return;
1108
1109	WARN_ON(!list_empty(&crif->nexthop_list));
1110	kfree(objp: crif);
1111	}
1112
1113	static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router,
1114	struct mlxsw_sp_crif *crif)
1115	{
1116	return rhashtable_insert_fast(ht: &router->crif_ht, obj: &crif->ht_node,
1117	params: mlxsw_sp_crif_ht_params);
1118	}
1119
1120	static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router,
1121	struct mlxsw_sp_crif *crif)
1122	{
1123	rhashtable_remove_fast(ht: &router->crif_ht, obj: &crif->ht_node,
1124	params: mlxsw_sp_crif_ht_params);
1125	}
1126
1127	static struct mlxsw_sp_crif *
1128	mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router,
1129	const struct net_device *dev)
1130	{
1131	struct mlxsw_sp_crif_key key = {
1132	.dev = (struct net_device *)dev,
1133	};
1134
1135	return rhashtable_lookup_fast(ht: &router->crif_ht, key: &key,
1136	params: mlxsw_sp_crif_ht_params);
1137	}
1138
1139	static struct mlxsw_sp_rif *
1140	mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1141	const struct mlxsw_sp_rif_params *params,
1142	struct netlink_ext_ack *extack);
1143
1144	static struct mlxsw_sp_rif_ipip_lb *
1145	mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1146	enum mlxsw_sp_ipip_type ipipt,
1147	struct net_device *ol_dev,
1148	struct netlink_ext_ack *extack)
1149	{
1150	struct mlxsw_sp_rif_params_ipip_lb lb_params;
1151	const struct mlxsw_sp_ipip_ops *ipip_ops;
1152	struct mlxsw_sp_rif *rif;
1153
1154	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1155	lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1156	.common.dev = ol_dev,
1157	.common.lag = false,
1158	.common.double_entry = ipip_ops->double_rif_entry,
1159	.lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1160	};
1161
1162	rif = mlxsw_sp_rif_create(mlxsw_sp, params: &lb_params.common, extack);
1163	if (IS_ERR(ptr: rif))
1164	return ERR_CAST(ptr: rif);
1165	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1166	}
1167
1168	static struct mlxsw_sp_ipip_entry *
1169	mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1170	enum mlxsw_sp_ipip_type ipipt,
1171	struct net_device *ol_dev)
1172	{
1173	const struct mlxsw_sp_ipip_ops *ipip_ops;
1174	struct mlxsw_sp_ipip_entry *ipip_entry;
1175	struct mlxsw_sp_ipip_entry *ret = NULL;
1176	int err;
1177
1178	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1179	ipip_entry = kzalloc(size: sizeof(*ipip_entry), GFP_KERNEL);
1180	if (!ipip_entry)
1181	return ERR_PTR(error: -ENOMEM);
1182
1183	ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1184	ol_dev, NULL);
1185	if (IS_ERR(ptr: ipip_entry->ol_lb)) {
1186	ret = ERR_CAST(ptr: ipip_entry->ol_lb);
1187	goto err_ol_ipip_lb_create;
1188	}
1189
1190	ipip_entry->ipipt = ipipt;
1191	ipip_entry->ol_dev = ol_dev;
1192	ipip_entry->parms = ipip_ops->parms_init(ol_dev);
1193
1194	err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry);
1195	if (err) {
1196	ret = ERR_PTR(error: err);
1197	goto err_rem_ip_addr_set;
1198	}
1199
1200	return ipip_entry;
1201
1202	err_rem_ip_addr_set:
1203	mlxsw_sp_rif_destroy(rif: &ipip_entry->ol_lb->common);
1204	err_ol_ipip_lb_create:
1205	kfree(objp: ipip_entry);
1206	return ret;
1207	}
1208
1209	static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp,
1210	struct mlxsw_sp_ipip_entry *ipip_entry)
1211	{
1212	const struct mlxsw_sp_ipip_ops *ipip_ops =
1213	mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1214
1215	ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry);
1216	mlxsw_sp_rif_destroy(rif: &ipip_entry->ol_lb->common);
1217	kfree(objp: ipip_entry);
1218	}
1219
1220	static bool
1221	mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1222	const enum mlxsw_sp_l3proto ul_proto,
1223	union mlxsw_sp_l3addr saddr,
1224	u32 ul_tb_id,
1225	struct mlxsw_sp_ipip_entry *ipip_entry)
1226	{
1227	u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev: ipip_entry->ol_dev);
1228	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1229	union mlxsw_sp_l3addr tun_saddr;
1230
1231	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1232	return false;
1233
1234	tun_saddr = mlxsw_sp_ipip_netdev_saddr(proto: ul_proto, ol_dev: ipip_entry->ol_dev);
1235	return tun_ul_tb_id == ul_tb_id &&
1236	mlxsw_sp_l3addr_eq(addr1: &tun_saddr, addr2: &saddr);
1237	}
1238
1239	static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp,
1240	enum mlxsw_sp_ipip_type ipipt)
1241	{
1242	const struct mlxsw_sp_ipip_ops *ipip_ops;
1243
1244	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1245
1246	/* Not all tunnels require to increase the default pasing depth
1247	* (96 bytes).
1248	*/
1249	if (ipip_ops->inc_parsing_depth)
1250	return mlxsw_sp_parsing_depth_inc(mlxsw_sp);
1251
1252	return 0;
1253	}
1254
1255	static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp,
1256	enum mlxsw_sp_ipip_type ipipt)
1257	{
1258	const struct mlxsw_sp_ipip_ops *ipip_ops =
1259	mlxsw_sp->router->ipip_ops_arr[ipipt];
1260
1261	if (ipip_ops->inc_parsing_depth)
1262	mlxsw_sp_parsing_depth_dec(mlxsw_sp);
1263	}
1264
1265	static int
1266	mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1267	struct mlxsw_sp_fib_entry *fib_entry,
1268	struct mlxsw_sp_ipip_entry *ipip_entry)
1269	{
1270	u32 tunnel_index;
1271	int err;
1272
1273	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1274	entry_count: 1, p_entry_index: &tunnel_index);
1275	if (err)
1276	return err;
1277
1278	err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp,
1279	ipipt: ipip_entry->ipipt);
1280	if (err)
1281	goto err_parsing_depth_inc;
1282
1283	ipip_entry->decap_fib_entry = fib_entry;
1284	fib_entry->decap.ipip_entry = ipip_entry;
1285	fib_entry->decap.tunnel_index = tunnel_index;
1286
1287	return 0;
1288
1289	err_parsing_depth_inc:
1290	mlxsw_sp_kvdl_free(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, entry_count: 1,
1291	entry_index: fib_entry->decap.tunnel_index);
1292	return err;
1293	}
1294
1295	static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1296	struct mlxsw_sp_fib_entry *fib_entry)
1297	{
1298	enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt;
1299
1300	/* Unlink this node from the IPIP entry that it's the decap entry of. */
1301	fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1302	fib_entry->decap.ipip_entry = NULL;
1303	mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt);
1304	mlxsw_sp_kvdl_free(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1305	entry_count: 1, entry_index: fib_entry->decap.tunnel_index);
1306	}
1307
1308	static struct mlxsw_sp_fib_node *
1309	mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1310	size_t addr_len, unsigned char prefix_len);
1311	static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1312	struct mlxsw_sp_fib_entry *fib_entry);
1313
1314	static void
1315	mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1316	struct mlxsw_sp_ipip_entry *ipip_entry)
1317	{
1318	struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1319
1320	mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1321	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1322
1323	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1324	}
1325
1326	static void
1327	mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1328	struct mlxsw_sp_ipip_entry *ipip_entry,
1329	struct mlxsw_sp_fib_entry *decap_fib_entry)
1330	{
1331	if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry: decap_fib_entry,
1332	ipip_entry))
1333	return;
1334	decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1335
1336	if (mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry: decap_fib_entry))
1337	mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1338	}
1339
1340	static struct mlxsw_sp_fib_entry *
1341	mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1342	enum mlxsw_sp_l3proto proto,
1343	const union mlxsw_sp_l3addr *addr,
1344	enum mlxsw_sp_fib_entry_type type)
1345	{
1346	struct mlxsw_sp_fib_node *fib_node;
1347	unsigned char addr_prefix_len;
1348	struct mlxsw_sp_fib *fib;
1349	struct mlxsw_sp_vr *vr;
1350	const void *addrp;
1351	size_t addr_len;
1352	u32 addr4;
1353
1354	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1355	if (!vr)
1356	return NULL;
1357	fib = mlxsw_sp_vr_fib(vr, proto);
1358
1359	switch (proto) {
1360	case MLXSW_SP_L3_PROTO_IPV4:
1361	addr4 = be32_to_cpu(addr->addr4);
1362	addrp = &addr4;
1363	addr_len = 4;
1364	addr_prefix_len = 32;
1365	break;
1366	case MLXSW_SP_L3_PROTO_IPV6:
1367	addrp = &addr->addr6;
1368	addr_len = 16;
1369	addr_prefix_len = 128;
1370	break;
1371	default:
1372	WARN_ON(1);
1373	return NULL;
1374	}
1375
1376	fib_node = mlxsw_sp_fib_node_lookup(fib, addr: addrp, addr_len,
1377	prefix_len: addr_prefix_len);
1378	if (!fib_node || fib_node->fib_entry->type != type)
1379	return NULL;
1380
1381	return fib_node->fib_entry;
1382	}
1383
1384	/* Given an IPIP entry, find the corresponding decap route. */
1385	static struct mlxsw_sp_fib_entry *
1386	mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1387	struct mlxsw_sp_ipip_entry *ipip_entry)
1388	{
1389	static struct mlxsw_sp_fib_node *fib_node;
1390	const struct mlxsw_sp_ipip_ops *ipip_ops;
1391	unsigned char saddr_prefix_len;
1392	union mlxsw_sp_l3addr saddr;
1393	struct mlxsw_sp_fib *ul_fib;
1394	struct mlxsw_sp_vr *ul_vr;
1395	const void *saddrp;
1396	size_t saddr_len;
1397	u32 ul_tb_id;
1398	u32 saddr4;
1399
1400	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1401
1402	ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev: ipip_entry->ol_dev);
1403	ul_vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id: ul_tb_id);
1404	if (!ul_vr)
1405	return NULL;
1406
1407	ul_fib = mlxsw_sp_vr_fib(vr: ul_vr, proto: ipip_ops->ul_proto);
1408	saddr = mlxsw_sp_ipip_netdev_saddr(proto: ipip_ops->ul_proto,
1409	ol_dev: ipip_entry->ol_dev);
1410
1411	switch (ipip_ops->ul_proto) {
1412	case MLXSW_SP_L3_PROTO_IPV4:
1413	saddr4 = be32_to_cpu(saddr.addr4);
1414	saddrp = &saddr4;
1415	saddr_len = 4;
1416	saddr_prefix_len = 32;
1417	break;
1418	case MLXSW_SP_L3_PROTO_IPV6:
1419	saddrp = &saddr.addr6;
1420	saddr_len = 16;
1421	saddr_prefix_len = 128;
1422	break;
1423	default:
1424	WARN_ON(1);
1425	return NULL;
1426	}
1427
1428	fib_node = mlxsw_sp_fib_node_lookup(fib: ul_fib, addr: saddrp, addr_len: saddr_len,
1429	prefix_len: saddr_prefix_len);
1430	if (!fib_node ||
1431	fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1432	return NULL;
1433
1434	return fib_node->fib_entry;
1435	}
1436
1437	static struct mlxsw_sp_ipip_entry *
1438	mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1439	enum mlxsw_sp_ipip_type ipipt,
1440	struct net_device *ol_dev)
1441	{
1442	struct mlxsw_sp_ipip_entry *ipip_entry;
1443
1444	ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1445	if (IS_ERR(ptr: ipip_entry))
1446	return ipip_entry;
1447
1448	list_add_tail(new: &ipip_entry->ipip_list_node,
1449	head: &mlxsw_sp->router->ipip_list);
1450
1451	return ipip_entry;
1452	}
1453
1454	static void
1455	mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1456	struct mlxsw_sp_ipip_entry *ipip_entry)
1457	{
1458	list_del(entry: &ipip_entry->ipip_list_node);
1459	mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry);
1460	}
1461
1462	static bool
1463	mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1464	const struct net_device *ul_dev,
1465	enum mlxsw_sp_l3proto ul_proto,
1466	union mlxsw_sp_l3addr ul_dip,
1467	struct mlxsw_sp_ipip_entry *ipip_entry)
1468	{
1469	u32 ul_tb_id = l3mdev_fib_table(dev: ul_dev) ? : RT_TABLE_MAIN;
1470	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1471
1472	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1473	return false;
1474
1475	return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr: ul_dip,
1476	ul_tb_id, ipip_entry);
1477	}
1478
1479	/* Given decap parameters, find the corresponding IPIP entry. */
1480	static struct mlxsw_sp_ipip_entry *
1481	mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1482	enum mlxsw_sp_l3proto ul_proto,
1483	union mlxsw_sp_l3addr ul_dip)
1484	{
1485	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1486	struct net_device *ul_dev;
1487
1488	rcu_read_lock();
1489
1490	ul_dev = dev_get_by_index_rcu(net: mlxsw_sp_net(mlxsw_sp), ifindex: ul_dev_ifindex);
1491	if (!ul_dev)
1492	goto out_unlock;
1493
1494	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1495	ipip_list_node)
1496	if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1497	ul_proto, ul_dip,
1498	ipip_entry))
1499	goto out_unlock;
1500
1501	rcu_read_unlock();
1502
1503	return NULL;
1504
1505	out_unlock:
1506	rcu_read_unlock();
1507	return ipip_entry;
1508	}
1509
1510	static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1511	const struct net_device *dev,
1512	enum mlxsw_sp_ipip_type *p_type)
1513	{
1514	struct mlxsw_sp_router *router = mlxsw_sp->router;
1515	const struct mlxsw_sp_ipip_ops *ipip_ops;
1516	enum mlxsw_sp_ipip_type ipipt;
1517
1518	for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1519	ipip_ops = router->ipip_ops_arr[ipipt];
1520	if (dev->type == ipip_ops->dev_type) {
1521	if (p_type)
1522	*p_type = ipipt;
1523	return true;
1524	}
1525	}
1526	return false;
1527	}
1528
1529	static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1530	const struct net_device *dev)
1531	{
1532	return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1533	}
1534
1535	static struct mlxsw_sp_ipip_entry *
1536	mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1537	const struct net_device *ol_dev)
1538	{
1539	struct mlxsw_sp_ipip_entry *ipip_entry;
1540
1541	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1542	ipip_list_node)
1543	if (ipip_entry->ol_dev == ol_dev)
1544	return ipip_entry;
1545
1546	return NULL;
1547	}
1548
1549	static struct mlxsw_sp_ipip_entry *
1550	mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1551	const struct net_device *ul_dev,
1552	struct mlxsw_sp_ipip_entry *start)
1553	{
1554	struct mlxsw_sp_ipip_entry *ipip_entry;
1555
1556	ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1557	ipip_list_node);
1558	list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1559	ipip_list_node) {
1560	struct net_device *ol_dev = ipip_entry->ol_dev;
1561	struct net_device *ipip_ul_dev;
1562
1563	rcu_read_lock();
1564	ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1565	rcu_read_unlock();
1566
1567	if (ipip_ul_dev == ul_dev)
1568	return ipip_entry;
1569	}
1570
1571	return NULL;
1572	}
1573
1574	static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1575	const struct net_device *dev)
1576	{
1577	return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, ul_dev: dev, NULL);
1578	}
1579
1580	static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1581	const struct net_device *ol_dev,
1582	enum mlxsw_sp_ipip_type ipipt)
1583	{
1584	const struct mlxsw_sp_ipip_ops *ops
1585	= mlxsw_sp->router->ipip_ops_arr[ipipt];
1586
1587	return ops->can_offload(mlxsw_sp, ol_dev);
1588	}
1589
1590	static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1591	struct net_device *ol_dev)
1592	{
1593	enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1594	struct mlxsw_sp_ipip_entry *ipip_entry;
1595	enum mlxsw_sp_l3proto ul_proto;
1596	union mlxsw_sp_l3addr saddr;
1597	u32 ul_tb_id;
1598
1599	mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev: ol_dev, p_type: &ipipt);
1600	if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1601	ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1602	ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1603	saddr = mlxsw_sp_ipip_netdev_saddr(proto: ul_proto, ol_dev);
1604	if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1605	saddr, ul_tb_id,
1606	NULL)) {
1607	ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1608	ol_dev);
1609	if (IS_ERR(ptr: ipip_entry))
1610	return PTR_ERR(ptr: ipip_entry);
1611	}
1612	}
1613
1614	return 0;
1615	}
1616
1617	static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1618	struct net_device *ol_dev)
1619	{
1620	struct mlxsw_sp_ipip_entry *ipip_entry;
1621
1622	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1623	if (ipip_entry)
1624	mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1625	}
1626
1627	static void
1628	mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1629	struct mlxsw_sp_ipip_entry *ipip_entry)
1630	{
1631	struct mlxsw_sp_fib_entry *decap_fib_entry;
1632
1633	decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1634	if (decap_fib_entry)
1635	mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1636	decap_fib_entry);
1637	}
1638
1639	static int
1640	mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1641	u16 ul_rif_id, bool enable)
1642	{
1643	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1644	struct net_device *dev = mlxsw_sp_rif_dev(rif: &lb_rif->common);
1645	enum mlxsw_reg_ritr_loopback_ipip_options ipip_options;
1646	struct mlxsw_sp_rif *rif = &lb_rif->common;
1647	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1648	char ritr_pl[MLXSW_REG_RITR_LEN];
1649	struct in6_addr *saddr6;
1650	u32 saddr4;
1651
1652	ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET;
1653	switch (lb_cf.ul_protocol) {
1654	case MLXSW_SP_L3_PROTO_IPV4:
1655	saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1656	mlxsw_reg_ritr_pack(payload: ritr_pl, enable, type: MLXSW_REG_RITR_LOOPBACK_IF,
1657	rif: rif->rif_index, vr_id: rif->vr_id, mtu: dev->mtu);
1658	mlxsw_reg_ritr_loopback_ipip4_pack(payload: ritr_pl, ipip_type: lb_cf.lb_ipipt,
1659	options: ipip_options, uvr_id: ul_vr_id,
1660	underlay_rif: ul_rif_id, usip: saddr4,
1661	gre_key: lb_cf.okey);
1662	break;
1663
1664	case MLXSW_SP_L3_PROTO_IPV6:
1665	saddr6 = &lb_cf.saddr.addr6;
1666	mlxsw_reg_ritr_pack(payload: ritr_pl, enable, type: MLXSW_REG_RITR_LOOPBACK_IF,
1667	rif: rif->rif_index, vr_id: rif->vr_id, mtu: dev->mtu);
1668	mlxsw_reg_ritr_loopback_ipip6_pack(payload: ritr_pl, ipip_type: lb_cf.lb_ipipt,
1669	options: ipip_options, uvr_id: ul_vr_id,
1670	underlay_rif: ul_rif_id, usip: saddr6,
1671	gre_key: lb_cf.okey);
1672	break;
1673	}
1674
1675	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
1676	}
1677
1678	static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1679	struct net_device *ol_dev)
1680	{
1681	struct mlxsw_sp_ipip_entry *ipip_entry;
1682	struct mlxsw_sp_rif_ipip_lb *lb_rif;
1683	int err = 0;
1684
1685	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1686	if (ipip_entry) {
1687	lb_rif = ipip_entry->ol_lb;
1688	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr_id: lb_rif->ul_vr_id,
1689	ul_rif_id: lb_rif->ul_rif_id, enable: true);
1690	if (err)
1691	goto out;
1692	lb_rif->common.mtu = ol_dev->mtu;
1693	}
1694
1695	out:
1696	return err;
1697	}
1698
1699	static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1700	struct net_device *ol_dev)
1701	{
1702	struct mlxsw_sp_ipip_entry *ipip_entry;
1703
1704	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1705	if (ipip_entry)
1706	mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1707	}
1708
1709	static void
1710	mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1711	struct mlxsw_sp_ipip_entry *ipip_entry)
1712	{
1713	if (ipip_entry->decap_fib_entry)
1714	mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1715	}
1716
1717	static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1718	struct net_device *ol_dev)
1719	{
1720	struct mlxsw_sp_ipip_entry *ipip_entry;
1721
1722	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1723	if (ipip_entry)
1724	mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1725	}
1726
1727	static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1728	struct mlxsw_sp_rif *rif);
1729
1730	static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp,
1731	struct mlxsw_sp_rif *old_rif,
1732	struct mlxsw_sp_rif *new_rif,
1733	bool migrate_nhs)
1734	{
1735	struct mlxsw_sp_crif *crif = old_rif->crif;
1736	struct mlxsw_sp_crif mock_crif = {};
1737
1738	if (migrate_nhs)
1739	mlxsw_sp_nexthop_rif_update(mlxsw_sp, rif: new_rif);
1740
1741	/* Plant a mock CRIF so that destroying the old RIF doesn't unoffload
1742	* our nexthops and IPIP tunnels, and doesn't sever the crif->rif link.
1743	*/
1744	mlxsw_sp_crif_init(crif: &mock_crif, dev: crif->key.dev);
1745	old_rif->crif = &mock_crif;
1746	mock_crif.rif = old_rif;
1747	mlxsw_sp_rif_destroy(rif: old_rif);
1748	}
1749
1750	static int
1751	mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1752	struct mlxsw_sp_ipip_entry *ipip_entry,
1753	bool keep_encap,
1754	struct netlink_ext_ack *extack)
1755	{
1756	struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1757	struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1758
1759	new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1760	ipipt: ipip_entry->ipipt,
1761	ol_dev: ipip_entry->ol_dev,
1762	extack);
1763	if (IS_ERR(ptr: new_lb_rif))
1764	return PTR_ERR(ptr: new_lb_rif);
1765	ipip_entry->ol_lb = new_lb_rif;
1766
1767	mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif: &old_lb_rif->common,
1768	new_rif: &new_lb_rif->common, migrate_nhs: keep_encap);
1769	return 0;
1770	}
1771
1772	/**
1773	* __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1774	* @mlxsw_sp: mlxsw_sp.
1775	* @ipip_entry: IPIP entry.
1776	* @recreate_loopback: Recreates the associated loopback RIF.
1777	* @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1778	* relevant when recreate_loopback is true.
1779	* @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1780	* is only relevant when recreate_loopback is false.
1781	* @extack: extack.
1782	*
1783	* Return: Non-zero value on failure.
1784	*/
1785	int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1786	struct mlxsw_sp_ipip_entry *ipip_entry,
1787	bool recreate_loopback,
1788	bool keep_encap,
1789	bool update_nexthops,
1790	struct netlink_ext_ack *extack)
1791	{
1792	int err;
1793
1794	/* RIFs can't be edited, so to update loopback, we need to destroy and
1795	* recreate it. That creates a window of opportunity where RALUE and
1796	* RATR registers end up referencing a RIF that's already gone. RATRs
1797	* are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1798	* of RALUE, demote the decap route back.
1799	*/
1800	if (ipip_entry->decap_fib_entry)
1801	mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1802
1803	if (recreate_loopback) {
1804	err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1805	keep_encap, extack);
1806	if (err)
1807	return err;
1808	} else if (update_nexthops) {
1809	mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1810	rif: &ipip_entry->ol_lb->common);
1811	}
1812
1813	if (ipip_entry->ol_dev->flags & IFF_UP)
1814	mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1815
1816	return 0;
1817	}
1818
1819	static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1820	struct net_device *ol_dev,
1821	struct netlink_ext_ack *extack)
1822	{
1823	struct mlxsw_sp_ipip_entry *ipip_entry =
1824	mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1825
1826	if (!ipip_entry)
1827	return 0;
1828
1829	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1830	recreate_loopback: true, keep_encap: false, update_nexthops: false, extack);
1831	}
1832
1833	static int
1834	mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1835	struct mlxsw_sp_ipip_entry *ipip_entry,
1836	struct net_device *ul_dev,
1837	bool *demote_this,
1838	struct netlink_ext_ack *extack)
1839	{
1840	u32 ul_tb_id = l3mdev_fib_table(dev: ul_dev) ? : RT_TABLE_MAIN;
1841	enum mlxsw_sp_l3proto ul_proto;
1842	union mlxsw_sp_l3addr saddr;
1843
1844	/* Moving underlay to a different VRF might cause local address
1845	* conflict, and the conflicting tunnels need to be demoted.
1846	*/
1847	ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1848	saddr = mlxsw_sp_ipip_netdev_saddr(proto: ul_proto, ol_dev: ipip_entry->ol_dev);
1849	if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1850	saddr, ul_tb_id,
1851	except: ipip_entry)) {
1852	*demote_this = true;
1853	return 0;
1854	}
1855
1856	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1857	recreate_loopback: true, keep_encap: true, update_nexthops: false, extack);
1858	}
1859
1860	static int
1861	mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1862	struct mlxsw_sp_ipip_entry *ipip_entry,
1863	struct net_device *ul_dev)
1864	{
1865	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1866	recreate_loopback: false, keep_encap: false, update_nexthops: true, NULL);
1867	}
1868
1869	static int
1870	mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1871	struct mlxsw_sp_ipip_entry *ipip_entry,
1872	struct net_device *ul_dev)
1873	{
1874	/* A down underlay device causes encapsulated packets to not be
1875	* forwarded, but decap still works. So refresh next hops without
1876	* touching anything else.
1877	*/
1878	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1879	recreate_loopback: false, keep_encap: false, update_nexthops: true, NULL);
1880	}
1881
1882	static int
1883	mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1884	struct net_device *ol_dev,
1885	struct netlink_ext_ack *extack)
1886	{
1887	const struct mlxsw_sp_ipip_ops *ipip_ops;
1888	struct mlxsw_sp_ipip_entry *ipip_entry;
1889	int err;
1890
1891	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1892	if (!ipip_entry)
1893	/* A change might make a tunnel eligible for offloading, but
1894	* that is currently not implemented. What falls to slow path
1895	* stays there.
1896	*/
1897	return 0;
1898
1899	/* A change might make a tunnel not eligible for offloading. */
1900	if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1901	ipipt: ipip_entry->ipipt)) {
1902	mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1903	return 0;
1904	}
1905
1906	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1907	err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1908	return err;
1909	}
1910
1911	void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1912	struct mlxsw_sp_ipip_entry *ipip_entry)
1913	{
1914	struct net_device *ol_dev = ipip_entry->ol_dev;
1915
1916	if (ol_dev->flags & IFF_UP)
1917	mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1918	mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1919	}
1920
1921	/* The configuration where several tunnels have the same local address in the
1922	* same underlay table needs special treatment in the HW. That is currently not
1923	* implemented in the driver. This function finds and demotes the first tunnel
1924	* with a given source address, except the one passed in the argument
1925	* `except'.
1926	*/
1927	bool
1928	mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1929	enum mlxsw_sp_l3proto ul_proto,
1930	union mlxsw_sp_l3addr saddr,
1931	u32 ul_tb_id,
1932	const struct mlxsw_sp_ipip_entry *except)
1933	{
1934	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1935
1936	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1937	ipip_list_node) {
1938	if (ipip_entry != except &&
1939	mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1940	ul_tb_id, ipip_entry)) {
1941	mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1942	return true;
1943	}
1944	}
1945
1946	return false;
1947	}
1948
1949	static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1950	struct net_device *ul_dev)
1951	{
1952	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1953
1954	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1955	ipip_list_node) {
1956	struct net_device *ol_dev = ipip_entry->ol_dev;
1957	struct net_device *ipip_ul_dev;
1958
1959	rcu_read_lock();
1960	ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1961	rcu_read_unlock();
1962	if (ipip_ul_dev == ul_dev)
1963	mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1964	}
1965	}
1966
1967	static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1968	struct net_device *ol_dev,
1969	unsigned long event,
1970	struct netdev_notifier_info *info)
1971	{
1972	struct netdev_notifier_changeupper_info *chup;
1973	struct netlink_ext_ack *extack;
1974	int err = 0;
1975
1976	switch (event) {
1977	case NETDEV_REGISTER:
1978	err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1979	break;
1980	case NETDEV_UNREGISTER:
1981	mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1982	break;
1983	case NETDEV_UP:
1984	mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1985	break;
1986	case NETDEV_DOWN:
1987	mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1988	break;
1989	case NETDEV_CHANGEUPPER:
1990	chup = container_of(info, typeof(*chup), info);
1991	extack = info->extack;
1992	if (netif_is_l3_master(dev: chup->upper_dev))
1993	err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1994	ol_dev,
1995	extack);
1996	break;
1997	case NETDEV_CHANGE:
1998	extack = info->extack;
1999	err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
2000	ol_dev, extack);
2001	break;
2002	case NETDEV_CHANGEMTU:
2003	err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
2004	break;
2005	}
2006	return err;
2007	}
2008
2009	static int
2010	__mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
2011	struct mlxsw_sp_ipip_entry *ipip_entry,
2012	struct net_device *ul_dev,
2013	bool *demote_this,
2014	unsigned long event,
2015	struct netdev_notifier_info *info)
2016	{
2017	struct netdev_notifier_changeupper_info *chup;
2018	struct netlink_ext_ack *extack;
2019
2020	switch (event) {
2021	case NETDEV_CHANGEUPPER:
2022	chup = container_of(info, typeof(*chup), info);
2023	extack = info->extack;
2024	if (netif_is_l3_master(dev: chup->upper_dev))
2025	return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
2026	ipip_entry,
2027	ul_dev,
2028	demote_this,
2029	extack);
2030	break;
2031
2032	case NETDEV_UP:
2033	return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
2034	ul_dev);
2035	case NETDEV_DOWN:
2036	return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
2037	ipip_entry,
2038	ul_dev);
2039	}
2040	return 0;
2041	}
2042
2043	static int
2044	mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
2045	struct net_device *ul_dev,
2046	unsigned long event,
2047	struct netdev_notifier_info *info)
2048	{
2049	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
2050	int err;
2051
2052	while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
2053	ul_dev,
2054	start: ipip_entry))) {
2055	struct mlxsw_sp_ipip_entry *prev;
2056	bool demote_this = false;
2057
2058	err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
2059	ul_dev, demote_this: &demote_this,
2060	event, info);
2061	if (err) {
2062	mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
2063	ul_dev);
2064	return err;
2065	}
2066
2067	if (demote_this) {
2068	if (list_is_first(list: &ipip_entry->ipip_list_node,
2069	head: &mlxsw_sp->router->ipip_list))
2070	prev = NULL;
2071	else
2072	/* This can't be cached from previous iteration,
2073	* because that entry could be gone now.
2074	*/
2075	prev = list_prev_entry(ipip_entry,
2076	ipip_list_node);
2077	mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
2078	ipip_entry = prev;
2079	}
2080	}
2081
2082	return 0;
2083	}
2084
2085	int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2086	enum mlxsw_sp_l3proto ul_proto,
2087	const union mlxsw_sp_l3addr *ul_sip,
2088	u32 tunnel_index)
2089	{
2090	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2091	struct mlxsw_sp_router *router = mlxsw_sp->router;
2092	struct mlxsw_sp_fib_entry *fib_entry;
2093	int err = 0;
2094
2095	mutex_lock(&mlxsw_sp->router->lock);
2096
2097	if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
2098	err = -EINVAL;
2099	goto out;
2100	}
2101
2102	router->nve_decap_config.ul_tb_id = ul_tb_id;
2103	router->nve_decap_config.tunnel_index = tunnel_index;
2104	router->nve_decap_config.ul_proto = ul_proto;
2105	router->nve_decap_config.ul_sip = *ul_sip;
2106	router->nve_decap_config.valid = true;
2107
2108	/* It is valid to create a tunnel with a local IP and only later
2109	* assign this IP address to a local interface
2110	*/
2111	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, tb_id: ul_tb_id,
2112	proto: ul_proto, addr: ul_sip,
2113	type);
2114	if (!fib_entry)
2115	goto out;
2116
2117	fib_entry->decap.tunnel_index = tunnel_index;
2118	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2119
2120	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2121	if (err)
2122	goto err_fib_entry_update;
2123
2124	goto out;
2125
2126	err_fib_entry_update:
2127	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2128	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2129	out:
2130	mutex_unlock(lock: &mlxsw_sp->router->lock);
2131	return err;
2132	}
2133
2134	void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2135	enum mlxsw_sp_l3proto ul_proto,
2136	const union mlxsw_sp_l3addr *ul_sip)
2137	{
2138	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2139	struct mlxsw_sp_router *router = mlxsw_sp->router;
2140	struct mlxsw_sp_fib_entry *fib_entry;
2141
2142	mutex_lock(&mlxsw_sp->router->lock);
2143
2144	if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2145	goto out;
2146
2147	router->nve_decap_config.valid = false;
2148
2149	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, tb_id: ul_tb_id,
2150	proto: ul_proto, addr: ul_sip,
2151	type);
2152	if (!fib_entry)
2153	goto out;
2154
2155	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2156	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2157	out:
2158	mutex_unlock(lock: &mlxsw_sp->router->lock);
2159	}
2160
2161	static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2162	u32 ul_tb_id,
2163	enum mlxsw_sp_l3proto ul_proto,
2164	const union mlxsw_sp_l3addr *ul_sip)
2165	{
2166	struct mlxsw_sp_router *router = mlxsw_sp->router;
2167
2168	return router->nve_decap_config.valid &&
2169	router->nve_decap_config.ul_tb_id == ul_tb_id &&
2170	router->nve_decap_config.ul_proto == ul_proto &&
2171	!memcmp(p: &router->nve_decap_config.ul_sip, q: ul_sip,
2172	size: sizeof(*ul_sip));
2173	}
2174
2175	struct mlxsw_sp_neigh_key {
2176	struct neighbour *n;
2177	};
2178
2179	struct mlxsw_sp_neigh_entry {
2180	struct list_head rif_list_node;
2181	struct rhash_head ht_node;
2182	struct mlxsw_sp_neigh_key key;
2183	u16 rif;
2184	bool connected;
2185	unsigned char ha[ETH_ALEN];
2186	struct list_head nexthop_list; /* list of nexthops using
2187	* this neigh entry
2188	*/
2189	struct list_head nexthop_neighs_list_node;
2190	unsigned int counter_index;
2191	bool counter_valid;
2192	};
2193
2194	static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2195	.key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2196	.head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2197	.key_len = sizeof(struct mlxsw_sp_neigh_key),
2198	};
2199
2200	struct mlxsw_sp_neigh_entry *
2201	mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2202	struct mlxsw_sp_neigh_entry *neigh_entry)
2203	{
2204	if (!neigh_entry) {
2205	if (list_empty(head: &rif->neigh_list))
2206	return NULL;
2207	else
2208	return list_first_entry(&rif->neigh_list,
2209	typeof(*neigh_entry),
2210	rif_list_node);
2211	}
2212	if (list_is_last(list: &neigh_entry->rif_list_node, head: &rif->neigh_list))
2213	return NULL;
2214	return list_next_entry(neigh_entry, rif_list_node);
2215	}
2216
2217	int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2218	{
2219	return neigh_entry->key.n->tbl->family;
2220	}
2221
2222	unsigned char *
2223	mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2224	{
2225	return neigh_entry->ha;
2226	}
2227
2228	u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2229	{
2230	struct neighbour *n;
2231
2232	n = neigh_entry->key.n;
2233	return ntohl(*((__be32 *) n->primary_key));
2234	}
2235
2236	struct in6_addr *
2237	mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2238	{
2239	struct neighbour *n;
2240
2241	n = neigh_entry->key.n;
2242	return (struct in6_addr *) &n->primary_key;
2243	}
2244
2245	int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2246	struct mlxsw_sp_neigh_entry *neigh_entry,
2247	u64 *p_counter)
2248	{
2249	if (!neigh_entry->counter_valid)
2250	return -EINVAL;
2251
2252	return mlxsw_sp_flow_counter_get(mlxsw_sp, counter_index: neigh_entry->counter_index,
2253	packets: p_counter, NULL);
2254	}
2255
2256	static struct mlxsw_sp_neigh_entry *
2257	mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2258	u16 rif)
2259	{
2260	struct mlxsw_sp_neigh_entry *neigh_entry;
2261
2262	neigh_entry = kzalloc(size: sizeof(*neigh_entry), GFP_KERNEL);
2263	if (!neigh_entry)
2264	return NULL;
2265
2266	neigh_entry->key.n = n;
2267	neigh_entry->rif = rif;
2268	INIT_LIST_HEAD(list: &neigh_entry->nexthop_list);
2269
2270	return neigh_entry;
2271	}
2272
2273	static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2274	{
2275	kfree(objp: neigh_entry);
2276	}
2277
2278	static int
2279	mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2280	struct mlxsw_sp_neigh_entry *neigh_entry)
2281	{
2282	return rhashtable_insert_fast(ht: &mlxsw_sp->router->neigh_ht,
2283	obj: &neigh_entry->ht_node,
2284	params: mlxsw_sp_neigh_ht_params);
2285	}
2286
2287	static void
2288	mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2289	struct mlxsw_sp_neigh_entry *neigh_entry)
2290	{
2291	rhashtable_remove_fast(ht: &mlxsw_sp->router->neigh_ht,
2292	obj: &neigh_entry->ht_node,
2293	params: mlxsw_sp_neigh_ht_params);
2294	}
2295
2296	static bool
2297	mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2298	struct mlxsw_sp_neigh_entry *neigh_entry)
2299	{
2300	struct devlink *devlink;
2301	const char *table_name;
2302
2303	switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2304	case AF_INET:
2305	table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2306	break;
2307	case AF_INET6:
2308	table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2309	break;
2310	default:
2311	WARN_ON(1);
2312	return false;
2313	}
2314
2315	devlink = priv_to_devlink(priv: mlxsw_sp->core);
2316	return devlink_dpipe_table_counter_enabled(devlink, table_name);
2317	}
2318
2319	static void
2320	mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2321	struct mlxsw_sp_neigh_entry *neigh_entry)
2322	{
2323	if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2324	return;
2325
2326	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, p_counter_index: &neigh_entry->counter_index))
2327	return;
2328
2329	neigh_entry->counter_valid = true;
2330	}
2331
2332	static void
2333	mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2334	struct mlxsw_sp_neigh_entry *neigh_entry)
2335	{
2336	if (!neigh_entry->counter_valid)
2337	return;
2338	mlxsw_sp_flow_counter_free(mlxsw_sp,
2339	counter_index: neigh_entry->counter_index);
2340	neigh_entry->counter_valid = false;
2341	}
2342
2343	static struct mlxsw_sp_neigh_entry *
2344	mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2345	{
2346	struct mlxsw_sp_neigh_entry *neigh_entry;
2347	struct mlxsw_sp_rif *rif;
2348	int err;
2349
2350	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: n->dev);
2351	if (!rif)
2352	return ERR_PTR(error: -EINVAL);
2353
2354	neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif: rif->rif_index);
2355	if (!neigh_entry)
2356	return ERR_PTR(error: -ENOMEM);
2357
2358	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2359	if (err)
2360	goto err_neigh_entry_insert;
2361
2362	mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2363	atomic_inc(v: &mlxsw_sp->router->neighs_update.neigh_count);
2364	list_add(new: &neigh_entry->rif_list_node, head: &rif->neigh_list);
2365
2366	return neigh_entry;
2367
2368	err_neigh_entry_insert:
2369	mlxsw_sp_neigh_entry_free(neigh_entry);
2370	return ERR_PTR(error: err);
2371	}
2372
2373	static void
2374	mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2375	struct mlxsw_sp_neigh_entry *neigh_entry)
2376	{
2377	list_del(entry: &neigh_entry->rif_list_node);
2378	atomic_dec(v: &mlxsw_sp->router->neighs_update.neigh_count);
2379	mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2380	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2381	mlxsw_sp_neigh_entry_free(neigh_entry);
2382	}
2383
2384	static struct mlxsw_sp_neigh_entry *
2385	mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2386	{
2387	struct mlxsw_sp_neigh_key key;
2388
2389	key.n = n;
2390	return rhashtable_lookup_fast(ht: &mlxsw_sp->router->neigh_ht,
2391	key: &key, params: mlxsw_sp_neigh_ht_params);
2392	}
2393
2394	static void
2395	mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2396	{
2397	unsigned long interval;
2398
2399	#if IS_ENABLED(CONFIG_IPV6)
2400	interval = min_t(unsigned long,
2401	NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2402	NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2403	#else
2404	interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2405	#endif
2406	mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(j: interval);
2407	}
2408
2409	static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2410	char *rauhtd_pl,
2411	int ent_index)
2412	{
2413	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2414	struct net_device *dev;
2415	struct neighbour *n;
2416	__be32 dipn;
2417	u32 dip;
2418	u16 rif;
2419
2420	mlxsw_reg_rauhtd_ent_ipv4_unpack(payload: rauhtd_pl, ent_index, p_rif: &rif, p_dip: &dip);
2421
2422	if (WARN_ON_ONCE(rif >= max_rifs))
2423	return;
2424	if (!mlxsw_sp->router->rifs[rif]) {
2425	dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2426	return;
2427	}
2428
2429	dipn = htonl(dip);
2430	dev = mlxsw_sp_rif_dev(rif: mlxsw_sp->router->rifs[rif]);
2431	n = neigh_lookup(tbl: &arp_tbl, pkey: &dipn, dev);
2432	if (!n)
2433	return;
2434
2435	netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2436	neigh_event_send(neigh: n, NULL);
2437	neigh_release(neigh: n);
2438	}
2439
2440	#if IS_ENABLED(CONFIG_IPV6)
2441	static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2442	char *rauhtd_pl,
2443	int rec_index)
2444	{
2445	struct net_device *dev;
2446	struct neighbour *n;
2447	struct in6_addr dip;
2448	u16 rif;
2449
2450	mlxsw_reg_rauhtd_ent_ipv6_unpack(payload: rauhtd_pl, rec_index, p_rif: &rif,
2451	p_dip: (char *) &dip);
2452
2453	if (!mlxsw_sp->router->rifs[rif]) {
2454	dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2455	return;
2456	}
2457
2458	dev = mlxsw_sp_rif_dev(rif: mlxsw_sp->router->rifs[rif]);
2459	n = neigh_lookup(tbl: &nd_tbl, pkey: &dip, dev);
2460	if (!n)
2461	return;
2462
2463	netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2464	neigh_event_send(neigh: n, NULL);
2465	neigh_release(neigh: n);
2466	}
2467	#else
2468	static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2469	char *rauhtd_pl,
2470	int rec_index)
2471	{
2472	}
2473	#endif
2474
2475	static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2476	char *rauhtd_pl,
2477	int rec_index)
2478	{
2479	u8 num_entries;
2480	int i;
2481
2482	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(buf: rauhtd_pl,
2483	index: rec_index);
2484	/* Hardware starts counting at 0, so add 1. */
2485	num_entries++;
2486
2487	/* Each record consists of several neighbour entries. */
2488	for (i = 0; i < num_entries; i++) {
2489	int ent_index;
2490
2491	ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2492	mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2493	ent_index);
2494	}
2495
2496	}
2497
2498	static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2499	char *rauhtd_pl,
2500	int rec_index)
2501	{
2502	/* One record contains one entry. */
2503	mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2504	rec_index);
2505	}
2506
2507	static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2508	char *rauhtd_pl, int rec_index)
2509	{
2510	switch (mlxsw_reg_rauhtd_rec_type_get(buf: rauhtd_pl, index: rec_index)) {
2511	case MLXSW_REG_RAUHTD_TYPE_IPV4:
2512	mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2513	rec_index);
2514	break;
2515	case MLXSW_REG_RAUHTD_TYPE_IPV6:
2516	mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2517	rec_index);
2518	break;
2519	}
2520	}
2521
2522	static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2523	{
2524	u8 num_rec, last_rec_index, num_entries;
2525
2526	num_rec = mlxsw_reg_rauhtd_num_rec_get(buf: rauhtd_pl);
2527	last_rec_index = num_rec - 1;
2528
2529	if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2530	return false;
2531	if (mlxsw_reg_rauhtd_rec_type_get(buf: rauhtd_pl, index: last_rec_index) ==
2532	MLXSW_REG_RAUHTD_TYPE_IPV6)
2533	return true;
2534
2535	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(buf: rauhtd_pl,
2536	index: last_rec_index);
2537	if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2538	return true;
2539	return false;
2540	}
2541
2542	static int
2543	__mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2544	char *rauhtd_pl,
2545	enum mlxsw_reg_rauhtd_type type)
2546	{
2547	int i, num_rec;
2548	int err;
2549
2550	/* Ensure the RIF we read from the device does not change mid-dump. */
2551	mutex_lock(&mlxsw_sp->router->lock);
2552	do {
2553	mlxsw_reg_rauhtd_pack(payload: rauhtd_pl, type);
2554	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(rauhtd),
2555	payload: rauhtd_pl);
2556	if (err) {
2557	dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2558	break;
2559	}
2560	num_rec = mlxsw_reg_rauhtd_num_rec_get(buf: rauhtd_pl);
2561	for (i = 0; i < num_rec; i++)
2562	mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2563	rec_index: i);
2564	} while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2565	mutex_unlock(lock: &mlxsw_sp->router->lock);
2566
2567	return err;
2568	}
2569
2570	static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2571	{
2572	enum mlxsw_reg_rauhtd_type type;
2573	char *rauhtd_pl;
2574	int err;
2575
2576	if (!atomic_read(v: &mlxsw_sp->router->neighs_update.neigh_count))
2577	return 0;
2578
2579	rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2580	if (!rauhtd_pl)
2581	return -ENOMEM;
2582
2583	type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2584	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2585	if (err)
2586	goto out;
2587
2588	type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2589	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2590	out:
2591	kfree(objp: rauhtd_pl);
2592	return err;
2593	}
2594
2595	static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2596	{
2597	struct mlxsw_sp_neigh_entry *neigh_entry;
2598
2599	mutex_lock(&mlxsw_sp->router->lock);
2600	list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2601	nexthop_neighs_list_node)
2602	/* If this neigh have nexthops, make the kernel think this neigh
2603	* is active regardless of the traffic.
2604	*/
2605	neigh_event_send(neigh: neigh_entry->key.n, NULL);
2606	mutex_unlock(lock: &mlxsw_sp->router->lock);
2607	}
2608
2609	static void
2610	mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2611	{
2612	unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2613
2614	mlxsw_core_schedule_dw(dwork: &mlxsw_sp->router->neighs_update.dw,
2615	delay: msecs_to_jiffies(m: interval));
2616	}
2617
2618	static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2619	{
2620	struct mlxsw_sp_router *router;
2621	int err;
2622
2623	router = container_of(work, struct mlxsw_sp_router,
2624	neighs_update.dw.work);
2625	err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp: router->mlxsw_sp);
2626	if (err)
2627	dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2628
2629	mlxsw_sp_router_neighs_update_nh(mlxsw_sp: router->mlxsw_sp);
2630
2631	mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp: router->mlxsw_sp);
2632	}
2633
2634	static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2635	{
2636	struct mlxsw_sp_neigh_entry *neigh_entry;
2637	struct mlxsw_sp_router *router;
2638
2639	router = container_of(work, struct mlxsw_sp_router,
2640	nexthop_probe_dw.work);
2641	/* Iterate over nexthop neighbours, find those who are unresolved and
2642	* send arp on them. This solves the chicken-egg problem when
2643	* the nexthop wouldn't get offloaded until the neighbor is resolved
2644	* but it wouldn't get resolved ever in case traffic is flowing in HW
2645	* using different nexthop.
2646	*/
2647	mutex_lock(&router->lock);
2648	list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2649	nexthop_neighs_list_node)
2650	if (!neigh_entry->connected)
2651	neigh_event_send(neigh: neigh_entry->key.n, NULL);
2652	mutex_unlock(lock: &router->lock);
2653
2654	mlxsw_core_schedule_dw(dwork: &router->nexthop_probe_dw,
2655	MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2656	}
2657
2658	static void
2659	mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2660	struct mlxsw_sp_neigh_entry *neigh_entry,
2661	bool removing, bool dead);
2662
2663	static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2664	{
2665	return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2666	MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2667	}
2668
2669	static int
2670	mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2671	struct mlxsw_sp_neigh_entry *neigh_entry,
2672	enum mlxsw_reg_rauht_op op)
2673	{
2674	struct neighbour *n = neigh_entry->key.n;
2675	u32 dip = ntohl(*((__be32 *) n->primary_key));
2676	char rauht_pl[MLXSW_REG_RAUHT_LEN];
2677
2678	mlxsw_reg_rauht_pack4(payload: rauht_pl, op, rif: neigh_entry->rif, mac: neigh_entry->ha,
2679	dip);
2680	if (neigh_entry->counter_valid)
2681	mlxsw_reg_rauht_pack_counter(payload: rauht_pl,
2682	counter_index: neigh_entry->counter_index);
2683	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(rauht), payload: rauht_pl);
2684	}
2685
2686	static int
2687	mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2688	struct mlxsw_sp_neigh_entry *neigh_entry,
2689	enum mlxsw_reg_rauht_op op)
2690	{
2691	struct neighbour *n = neigh_entry->key.n;
2692	char rauht_pl[MLXSW_REG_RAUHT_LEN];
2693	const char *dip = n->primary_key;
2694
2695	mlxsw_reg_rauht_pack6(payload: rauht_pl, op, rif: neigh_entry->rif, mac: neigh_entry->ha,
2696	dip);
2697	if (neigh_entry->counter_valid)
2698	mlxsw_reg_rauht_pack_counter(payload: rauht_pl,
2699	counter_index: neigh_entry->counter_index);
2700	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(rauht), payload: rauht_pl);
2701	}
2702
2703	bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2704	{
2705	struct neighbour *n = neigh_entry->key.n;
2706
2707	/* Packets with a link-local destination address are trapped
2708	* after LPM lookup and never reach the neighbour table, so
2709	* there is no need to program such neighbours to the device.
2710	*/
2711	if (ipv6_addr_type(addr: (struct in6_addr *) &n->primary_key) &
2712	IPV6_ADDR_LINKLOCAL)
2713	return true;
2714	return false;
2715	}
2716
2717	static void
2718	mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2719	struct mlxsw_sp_neigh_entry *neigh_entry,
2720	bool adding)
2721	{
2722	enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2723	int err;
2724
2725	if (!adding && !neigh_entry->connected)
2726	return;
2727	neigh_entry->connected = adding;
2728	if (neigh_entry->key.n->tbl->family == AF_INET) {
2729	err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2730	op);
2731	if (err)
2732	return;
2733	} else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2734	if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2735	return;
2736	err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2737	op);
2738	if (err)
2739	return;
2740	} else {
2741	WARN_ON_ONCE(1);
2742	return;
2743	}
2744
2745	if (adding)
2746	neigh_entry->key.n->flags |= NTF_OFFLOADED;
2747	else
2748	neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2749	}
2750
2751	void
2752	mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2753	struct mlxsw_sp_neigh_entry *neigh_entry,
2754	bool adding)
2755	{
2756	if (adding)
2757	mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2758	else
2759	mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2760	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, adding: true);
2761	}
2762
2763	struct mlxsw_sp_netevent_work {
2764	struct work_struct work;
2765	struct mlxsw_sp *mlxsw_sp;
2766	struct neighbour *n;
2767	};
2768
2769	static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2770	{
2771	struct mlxsw_sp_netevent_work *net_work =
2772	container_of(work, struct mlxsw_sp_netevent_work, work);
2773	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2774	struct mlxsw_sp_neigh_entry *neigh_entry;
2775	struct neighbour *n = net_work->n;
2776	unsigned char ha[ETH_ALEN];
2777	bool entry_connected;
2778	u8 nud_state, dead;
2779
2780	/* If these parameters are changed after we release the lock,
2781	* then we are guaranteed to receive another event letting us
2782	* know about it.
2783	*/
2784	read_lock_bh(&n->lock);
2785	memcpy(ha, n->ha, ETH_ALEN);
2786	nud_state = n->nud_state;
2787	dead = n->dead;
2788	read_unlock_bh(&n->lock);
2789
2790	mutex_lock(&mlxsw_sp->router->lock);
2791	mlxsw_sp_span_respin(mlxsw_sp);
2792
2793	entry_connected = nud_state & NUD_VALID && !dead;
2794	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2795	if (!entry_connected && !neigh_entry)
2796	goto out;
2797	if (!neigh_entry) {
2798	neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2799	if (IS_ERR(ptr: neigh_entry))
2800	goto out;
2801	}
2802
2803	if (neigh_entry->connected && entry_connected &&
2804	!memcmp(p: neigh_entry->ha, q: ha, ETH_ALEN))
2805	goto out;
2806
2807	memcpy(neigh_entry->ha, ha, ETH_ALEN);
2808	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, adding: entry_connected);
2809	mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, removing: !entry_connected,
2810	dead);
2811
2812	if (!neigh_entry->connected && list_empty(head: &neigh_entry->nexthop_list))
2813	mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2814
2815	out:
2816	mutex_unlock(lock: &mlxsw_sp->router->lock);
2817	neigh_release(neigh: n);
2818	kfree(objp: net_work);
2819	}
2820
2821	static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2822
2823	static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2824	{
2825	struct mlxsw_sp_netevent_work *net_work =
2826	container_of(work, struct mlxsw_sp_netevent_work, work);
2827	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2828
2829	mlxsw_sp_mp_hash_init(mlxsw_sp);
2830	kfree(objp: net_work);
2831	}
2832
2833	static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2834
2835	static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2836	{
2837	struct mlxsw_sp_netevent_work *net_work =
2838	container_of(work, struct mlxsw_sp_netevent_work, work);
2839	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2840
2841	__mlxsw_sp_router_init(mlxsw_sp);
2842	kfree(objp: net_work);
2843	}
2844
2845	static int mlxsw_sp_router_schedule_work(struct net *net,
2846	struct mlxsw_sp_router *router,
2847	struct neighbour *n,
2848	void (*cb)(struct work_struct *))
2849	{
2850	struct mlxsw_sp_netevent_work *net_work;
2851
2852	if (!net_eq(net1: net, net2: mlxsw_sp_net(mlxsw_sp: router->mlxsw_sp)))
2853	return NOTIFY_DONE;
2854
2855	net_work = kzalloc(size: sizeof(*net_work), GFP_ATOMIC);
2856	if (!net_work)
2857	return NOTIFY_BAD;
2858
2859	INIT_WORK(&net_work->work, cb);
2860	net_work->mlxsw_sp = router->mlxsw_sp;
2861	net_work->n = n;
2862	mlxsw_core_schedule_work(work: &net_work->work);
2863	return NOTIFY_DONE;
2864	}
2865
2866	static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev)
2867	{
2868	struct mlxsw_sp_port *mlxsw_sp_port;
2869
2870	rcu_read_lock();
2871	mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev);
2872	rcu_read_unlock();
2873	return !!mlxsw_sp_port;
2874	}
2875
2876	static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router,
2877	struct neighbour *n)
2878	{
2879	struct net *net;
2880
2881	net = neigh_parms_net(parms: n->parms);
2882
2883	/* Take a reference to ensure the neighbour won't be destructed until we
2884	* drop the reference in delayed work.
2885	*/
2886	neigh_clone(neigh: n);
2887	return mlxsw_sp_router_schedule_work(net, router, n,
2888	cb: mlxsw_sp_router_neigh_event_work);
2889	}
2890
2891	static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2892	unsigned long event, void *ptr)
2893	{
2894	struct mlxsw_sp_router *router;
2895	unsigned long interval;
2896	struct neigh_parms *p;
2897	struct neighbour *n;
2898
2899	router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2900
2901	switch (event) {
2902	case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2903	p = ptr;
2904
2905	/* We don't care about changes in the default table. */
2906	if (!p->dev || (p->tbl->family != AF_INET &&
2907	p->tbl->family != AF_INET6))
2908	return NOTIFY_DONE;
2909
2910	/* We are in atomic context and can't take RTNL mutex,
2911	* so use RCU variant to walk the device chain.
2912	*/
2913	if (!mlxsw_sp_dev_lower_is_port(dev: p->dev))
2914	return NOTIFY_DONE;
2915
2916	interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2917	router->neighs_update.interval = interval;
2918	break;
2919	case NETEVENT_NEIGH_UPDATE:
2920	n = ptr;
2921
2922	if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2923	return NOTIFY_DONE;
2924
2925	if (!mlxsw_sp_dev_lower_is_port(dev: n->dev))
2926	return NOTIFY_DONE;
2927
2928	return mlxsw_sp_router_schedule_neigh_work(router, n);
2929
2930	case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2931	case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2932	return mlxsw_sp_router_schedule_work(net: ptr, router, NULL,
2933	cb: mlxsw_sp_router_mp_hash_event_work);
2934
2935	case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2936	return mlxsw_sp_router_schedule_work(net: ptr, router, NULL,
2937	cb: mlxsw_sp_router_update_priority_work);
2938	}
2939
2940	return NOTIFY_DONE;
2941	}
2942
2943	static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2944	{
2945	int err;
2946
2947	err = rhashtable_init(ht: &mlxsw_sp->router->neigh_ht,
2948	params: &mlxsw_sp_neigh_ht_params);
2949	if (err)
2950	return err;
2951
2952	/* Initialize the polling interval according to the default
2953	* table.
2954	*/
2955	mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2956
2957	/* Create the delayed works for the activity_update */
2958	INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2959	mlxsw_sp_router_neighs_update_work);
2960	INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2961	mlxsw_sp_router_probe_unresolved_nexthops);
2962	atomic_set(v: &mlxsw_sp->router->neighs_update.neigh_count, i: 0);
2963	mlxsw_core_schedule_dw(dwork: &mlxsw_sp->router->neighs_update.dw, delay: 0);
2964	mlxsw_core_schedule_dw(dwork: &mlxsw_sp->router->nexthop_probe_dw, delay: 0);
2965	return 0;
2966	}
2967
2968	static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2969	{
2970	cancel_delayed_work_sync(dwork: &mlxsw_sp->router->neighs_update.dw);
2971	cancel_delayed_work_sync(dwork: &mlxsw_sp->router->nexthop_probe_dw);
2972	rhashtable_destroy(ht: &mlxsw_sp->router->neigh_ht);
2973	}
2974
2975	static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2976	struct mlxsw_sp_rif *rif)
2977	{
2978	struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2979
2980	list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2981	rif_list_node) {
2982	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, adding: false);
2983	mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2984	}
2985	}
2986
2987	struct mlxsw_sp_neigh_rif_made_sync {
2988	struct mlxsw_sp *mlxsw_sp;
2989	struct mlxsw_sp_rif *rif;
2990	int err;
2991	};
2992
2993	static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data)
2994	{
2995	struct mlxsw_sp_neigh_rif_made_sync *rms = data;
2996	int rc;
2997
2998	if (rms->err)
2999	return;
3000	if (n->dev != mlxsw_sp_rif_dev(rif: rms->rif))
3001	return;
3002	rc = mlxsw_sp_router_schedule_neigh_work(router: rms->mlxsw_sp->router, n);
3003	if (rc != NOTIFY_DONE)
3004	rms->err = -ENOMEM;
3005	}
3006
3007	static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
3008	struct mlxsw_sp_rif *rif)
3009	{
3010	struct mlxsw_sp_neigh_rif_made_sync rms = {
3011	.mlxsw_sp = mlxsw_sp,
3012	.rif = rif,
3013	};
3014
3015	neigh_for_each(tbl: &arp_tbl, cb: mlxsw_sp_neigh_rif_made_sync_each, cookie: &rms);
3016	if (rms.err)
3017	goto err_arp;
3018
3019	#if IS_ENABLED(CONFIG_IPV6)
3020	neigh_for_each(tbl: &nd_tbl, cb: mlxsw_sp_neigh_rif_made_sync_each, cookie: &rms);
3021	#endif
3022	if (rms.err)
3023	goto err_nd;
3024
3025	return 0;
3026
3027	err_nd:
3028	err_arp:
3029	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
3030	return rms.err;
3031	}
3032
3033	enum mlxsw_sp_nexthop_type {
3034	MLXSW_SP_NEXTHOP_TYPE_ETH,
3035	MLXSW_SP_NEXTHOP_TYPE_IPIP,
3036	};
3037
3038	enum mlxsw_sp_nexthop_action {
3039	/* Nexthop forwards packets to an egress RIF */
3040	MLXSW_SP_NEXTHOP_ACTION_FORWARD,
3041	/* Nexthop discards packets */
3042	MLXSW_SP_NEXTHOP_ACTION_DISCARD,
3043	/* Nexthop traps packets */
3044	MLXSW_SP_NEXTHOP_ACTION_TRAP,
3045	};
3046
3047	struct mlxsw_sp_nexthop_key {
3048	struct fib_nh *fib_nh;
3049	};
3050
3051	struct mlxsw_sp_nexthop {
3052	struct list_head neigh_list_node; /* member of neigh entry list */
3053	struct list_head crif_list_node;
3054	struct list_head router_list_node;
3055	struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
3056	* this nexthop belongs to
3057	*/
3058	struct rhash_head ht_node;
3059	struct neigh_table *neigh_tbl;
3060	struct mlxsw_sp_nexthop_key key;
3061	unsigned char gw_addr[sizeof(struct in6_addr)];
3062	int ifindex;
3063	int nh_weight;
3064	int norm_nh_weight;
3065	int num_adj_entries;
3066	struct mlxsw_sp_crif *crif;
3067	u8 should_offload:1, /* set indicates this nexthop should be written
3068	* to the adjacency table.
3069	*/
3070	offloaded:1, /* set indicates this nexthop was written to the
3071	* adjacency table.
3072	*/
3073	update:1; /* set indicates this nexthop should be updated in the
3074	* adjacency table (f.e., its MAC changed).
3075	*/
3076	enum mlxsw_sp_nexthop_action action;
3077	enum mlxsw_sp_nexthop_type type;
3078	union {
3079	struct mlxsw_sp_neigh_entry *neigh_entry;
3080	struct mlxsw_sp_ipip_entry *ipip_entry;
3081	};
3082	unsigned int counter_index;
3083	bool counter_valid;
3084	};
3085
3086	static struct net_device *
3087	mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh)
3088	{
3089	if (!nh->crif)
3090	return NULL;
3091	return nh->crif->key.dev;
3092	}
3093
3094	enum mlxsw_sp_nexthop_group_type {
3095	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
3096	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
3097	MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
3098	};
3099
3100	struct mlxsw_sp_nexthop_group_info {
3101	struct mlxsw_sp_nexthop_group *nh_grp;
3102	u32 adj_index;
3103	u16 ecmp_size;
3104	u16 count;
3105	int sum_norm_weight;
3106	u8 adj_index_valid:1,
3107	gateway:1, /* routes using the group use a gateway */
3108	is_resilient:1;
3109	struct list_head list; /* member in nh_res_grp_list */
3110	struct mlxsw_sp_nexthop nexthops[] __counted_by(count);
3111	};
3112
3113	static struct mlxsw_sp_rif *
3114	mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi)
3115	{
3116	struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif;
3117
3118	if (!crif)
3119	return NULL;
3120	return crif->rif;
3121	}
3122
3123	struct mlxsw_sp_nexthop_group_vr_key {
3124	u16 vr_id;
3125	enum mlxsw_sp_l3proto proto;
3126	};
3127
3128	struct mlxsw_sp_nexthop_group_vr_entry {
3129	struct list_head list; /* member in vr_list */
3130	struct rhash_head ht_node; /* member in vr_ht */
3131	refcount_t ref_count;
3132	struct mlxsw_sp_nexthop_group_vr_key key;
3133	};
3134
3135	struct mlxsw_sp_nexthop_group {
3136	struct rhash_head ht_node;
3137	struct list_head fib_list; /* list of fib entries that use this group */
3138	union {
3139	struct {
3140	struct fib_info *fi;
3141	} ipv4;
3142	struct {
3143	u32 id;
3144	} obj;
3145	};
3146	struct mlxsw_sp_nexthop_group_info *nhgi;
3147	struct list_head vr_list;
3148	struct rhashtable vr_ht;
3149	enum mlxsw_sp_nexthop_group_type type;
3150	bool can_destroy;
3151	};
3152
3153	void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
3154	struct mlxsw_sp_nexthop *nh)
3155	{
3156	struct devlink *devlink;
3157
3158	devlink = priv_to_devlink(priv: mlxsw_sp->core);
3159	if (!devlink_dpipe_table_counter_enabled(devlink,
3160	MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
3161	return;
3162
3163	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, p_counter_index: &nh->counter_index))
3164	return;
3165
3166	nh->counter_valid = true;
3167	}
3168
3169	void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
3170	struct mlxsw_sp_nexthop *nh)
3171	{
3172	if (!nh->counter_valid)
3173	return;
3174	mlxsw_sp_flow_counter_free(mlxsw_sp, counter_index: nh->counter_index);
3175	nh->counter_valid = false;
3176	}
3177
3178	int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
3179	struct mlxsw_sp_nexthop *nh, u64 *p_counter)
3180	{
3181	if (!nh->counter_valid)
3182	return -EINVAL;
3183
3184	return mlxsw_sp_flow_counter_get(mlxsw_sp, counter_index: nh->counter_index,
3185	packets: p_counter, NULL);
3186	}
3187
3188	struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
3189	struct mlxsw_sp_nexthop *nh)
3190	{
3191	if (!nh) {
3192	if (list_empty(head: &router->nexthop_list))
3193	return NULL;
3194	else
3195	return list_first_entry(&router->nexthop_list,
3196	typeof(*nh), router_list_node);
3197	}
3198	if (list_is_last(list: &nh->router_list_node, head: &router->nexthop_list))
3199	return NULL;
3200	return list_next_entry(nh, router_list_node);
3201	}
3202
3203	bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
3204	{
3205	return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3206	}
3207
3208	unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
3209	{
3210	if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
3211	!mlxsw_sp_nexthop_is_forward(nh))
3212	return NULL;
3213	return nh->neigh_entry->ha;
3214	}
3215
3216	int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
3217	u32 *p_adj_size, u32 *p_adj_hash_index)
3218	{
3219	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3220	u32 adj_hash_index = 0;
3221	int i;
3222
3223	if (!nh->offloaded || !nhgi->adj_index_valid)
3224	return -EINVAL;
3225
3226	*p_adj_index = nhgi->adj_index;
3227	*p_adj_size = nhgi->ecmp_size;
3228
3229	for (i = 0; i < nhgi->count; i++) {
3230	struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3231
3232	if (nh_iter == nh)
3233	break;
3234	if (nh_iter->offloaded)
3235	adj_hash_index += nh_iter->num_adj_entries;
3236	}
3237
3238	*p_adj_hash_index = adj_hash_index;
3239	return 0;
3240	}
3241
3242	struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
3243	{
3244	if (WARN_ON(!nh->crif))
3245	return NULL;
3246	return nh->crif->rif;
3247	}
3248
3249	bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
3250	{
3251	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3252	int i;
3253
3254	for (i = 0; i < nhgi->count; i++) {
3255	struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3256
3257	if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3258	return true;
3259	}
3260	return false;
3261	}
3262
3263	static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
3264	.key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
3265	.head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
3266	.key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
3267	.automatic_shrinking = true,
3268	};
3269
3270	static struct mlxsw_sp_nexthop_group_vr_entry *
3271	mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
3272	const struct mlxsw_sp_fib *fib)
3273	{
3274	struct mlxsw_sp_nexthop_group_vr_key key;
3275
3276	memset(&key, 0, sizeof(key));
3277	key.vr_id = fib->vr->id;
3278	key.proto = fib->proto;
3279	return rhashtable_lookup_fast(ht: &nh_grp->vr_ht, key: &key,
3280	params: mlxsw_sp_nexthop_group_vr_ht_params);
3281	}
3282
3283	static int
3284	mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
3285	const struct mlxsw_sp_fib *fib)
3286	{
3287	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3288	int err;
3289
3290	vr_entry = kzalloc(size: sizeof(*vr_entry), GFP_KERNEL);
3291	if (!vr_entry)
3292	return -ENOMEM;
3293
3294	vr_entry->key.vr_id = fib->vr->id;
3295	vr_entry->key.proto = fib->proto;
3296	refcount_set(r: &vr_entry->ref_count, n: 1);
3297
3298	err = rhashtable_insert_fast(ht: &nh_grp->vr_ht, obj: &vr_entry->ht_node,
3299	params: mlxsw_sp_nexthop_group_vr_ht_params);
3300	if (err)
3301	goto err_hashtable_insert;
3302
3303	list_add(new: &vr_entry->list, head: &nh_grp->vr_list);
3304
3305	return 0;
3306
3307	err_hashtable_insert:
3308	kfree(objp: vr_entry);
3309	return err;
3310	}
3311
3312	static void
3313	mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
3314	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
3315	{
3316	list_del(entry: &vr_entry->list);
3317	rhashtable_remove_fast(ht: &nh_grp->vr_ht, obj: &vr_entry->ht_node,
3318	params: mlxsw_sp_nexthop_group_vr_ht_params);
3319	kfree(objp: vr_entry);
3320	}
3321
3322	static int
3323	mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
3324	const struct mlxsw_sp_fib *fib)
3325	{
3326	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3327
3328	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3329	if (vr_entry) {
3330	refcount_inc(r: &vr_entry->ref_count);
3331	return 0;
3332	}
3333
3334	return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
3335	}
3336
3337	static void
3338	mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
3339	const struct mlxsw_sp_fib *fib)
3340	{
3341	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3342
3343	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3344	if (WARN_ON_ONCE(!vr_entry))
3345	return;
3346
3347	if (!refcount_dec_and_test(r: &vr_entry->ref_count))
3348	return;
3349
3350	mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
3351	}
3352
3353	struct mlxsw_sp_nexthop_group_cmp_arg {
3354	enum mlxsw_sp_nexthop_group_type type;
3355	union {
3356	struct fib_info *fi;
3357	struct mlxsw_sp_fib6_entry *fib6_entry;
3358	u32 id;
3359	};
3360	};
3361
3362	static bool
3363	mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3364	const struct in6_addr *gw, int ifindex,
3365	int weight)
3366	{
3367	int i;
3368
3369	for (i = 0; i < nh_grp->nhgi->count; i++) {
3370	const struct mlxsw_sp_nexthop *nh;
3371
3372	nh = &nh_grp->nhgi->nexthops[i];
3373	if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3374	ipv6_addr_equal(a1: gw, a2: (struct in6_addr *) nh->gw_addr))
3375	return true;
3376	}
3377
3378	return false;
3379	}
3380
3381	static bool
3382	mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3383	const struct mlxsw_sp_fib6_entry *fib6_entry)
3384	{
3385	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3386
3387	if (nh_grp->nhgi->count != fib6_entry->nrt6)
3388	return false;
3389
3390	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3391	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3392	struct in6_addr *gw;
3393	int ifindex, weight;
3394
3395	ifindex = fib6_nh->fib_nh_dev->ifindex;
3396	weight = fib6_nh->fib_nh_weight;
3397	gw = &fib6_nh->fib_nh_gw6;
3398	if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3399	weight))
3400	return false;
3401	}
3402
3403	return true;
3404	}
3405
3406	static int
3407	mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3408	{
3409	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3410	const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3411
3412	if (nh_grp->type != cmp_arg->type)
3413	return 1;
3414
3415	switch (cmp_arg->type) {
3416	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3417	return cmp_arg->fi != nh_grp->ipv4.fi;
3418	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3419	return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3420	fib6_entry: cmp_arg->fib6_entry);
3421	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3422	return cmp_arg->id != nh_grp->obj.id;
3423	default:
3424	WARN_ON(1);
3425	return 1;
3426	}
3427	}
3428
3429	static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3430	{
3431	const struct mlxsw_sp_nexthop_group *nh_grp = data;
3432	const struct mlxsw_sp_nexthop *nh;
3433	struct fib_info *fi;
3434	unsigned int val;
3435	int i;
3436
3437	switch (nh_grp->type) {
3438	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3439	fi = nh_grp->ipv4.fi;
3440	return jhash(key: &fi, length: sizeof(fi), initval: seed);
3441	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3442	val = nh_grp->nhgi->count;
3443	for (i = 0; i < nh_grp->nhgi->count; i++) {
3444	nh = &nh_grp->nhgi->nexthops[i];
3445	val ^= jhash(key: &nh->ifindex, length: sizeof(nh->ifindex), initval: seed);
3446	val ^= jhash(key: &nh->gw_addr, length: sizeof(nh->gw_addr), initval: seed);
3447	}
3448	return jhash(key: &val, length: sizeof(val), initval: seed);
3449	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3450	return jhash(key: &nh_grp->obj.id, length: sizeof(nh_grp->obj.id), initval: seed);
3451	default:
3452	WARN_ON(1);
3453	return 0;
3454	}
3455	}
3456
3457	static u32
3458	mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3459	{
3460	unsigned int val = fib6_entry->nrt6;
3461	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3462
3463	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3464	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3465	struct net_device *dev = fib6_nh->fib_nh_dev;
3466	struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3467
3468	val ^= jhash(key: &dev->ifindex, length: sizeof(dev->ifindex), initval: seed);
3469	val ^= jhash(key: gw, length: sizeof(*gw), initval: seed);
3470	}
3471
3472	return jhash(key: &val, length: sizeof(val), initval: seed);
3473	}
3474
3475	static u32
3476	mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3477	{
3478	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3479
3480	switch (cmp_arg->type) {
3481	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3482	return jhash(key: &cmp_arg->fi, length: sizeof(cmp_arg->fi), initval: seed);
3483	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3484	return mlxsw_sp_nexthop6_group_hash(fib6_entry: cmp_arg->fib6_entry, seed);
3485	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3486	return jhash(key: &cmp_arg->id, length: sizeof(cmp_arg->id), initval: seed);
3487	default:
3488	WARN_ON(1);
3489	return 0;
3490	}
3491	}
3492
3493	static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3494	.head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3495	.hashfn = mlxsw_sp_nexthop_group_hash,
3496	.obj_hashfn = mlxsw_sp_nexthop_group_hash_obj,
3497	.obj_cmpfn = mlxsw_sp_nexthop_group_cmp,
3498	};
3499
3500	static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3501	struct mlxsw_sp_nexthop_group *nh_grp)
3502	{
3503	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3504	!nh_grp->nhgi->gateway)
3505	return 0;
3506
3507	return rhashtable_insert_fast(ht: &mlxsw_sp->router->nexthop_group_ht,
3508	obj: &nh_grp->ht_node,
3509	params: mlxsw_sp_nexthop_group_ht_params);
3510	}
3511
3512	static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3513	struct mlxsw_sp_nexthop_group *nh_grp)
3514	{
3515	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3516	!nh_grp->nhgi->gateway)
3517	return;
3518
3519	rhashtable_remove_fast(ht: &mlxsw_sp->router->nexthop_group_ht,
3520	obj: &nh_grp->ht_node,
3521	params: mlxsw_sp_nexthop_group_ht_params);
3522	}
3523
3524	static struct mlxsw_sp_nexthop_group *
3525	mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3526	struct fib_info *fi)
3527	{
3528	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3529
3530	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3531	cmp_arg.fi = fi;
3532	return rhashtable_lookup_fast(ht: &mlxsw_sp->router->nexthop_group_ht,
3533	key: &cmp_arg,
3534	params: mlxsw_sp_nexthop_group_ht_params);
3535	}
3536
3537	static struct mlxsw_sp_nexthop_group *
3538	mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3539	struct mlxsw_sp_fib6_entry *fib6_entry)
3540	{
3541	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3542
3543	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3544	cmp_arg.fib6_entry = fib6_entry;
3545	return rhashtable_lookup_fast(ht: &mlxsw_sp->router->nexthop_group_ht,
3546	key: &cmp_arg,
3547	params: mlxsw_sp_nexthop_group_ht_params);
3548	}
3549
3550	static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3551	.key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3552	.head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3553	.key_len = sizeof(struct mlxsw_sp_nexthop_key),
3554	};
3555
3556	static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3557	struct mlxsw_sp_nexthop *nh)
3558	{
3559	return rhashtable_insert_fast(ht: &mlxsw_sp->router->nexthop_ht,
3560	obj: &nh->ht_node, params: mlxsw_sp_nexthop_ht_params);
3561	}
3562
3563	static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3564	struct mlxsw_sp_nexthop *nh)
3565	{
3566	rhashtable_remove_fast(ht: &mlxsw_sp->router->nexthop_ht, obj: &nh->ht_node,
3567	params: mlxsw_sp_nexthop_ht_params);
3568	}
3569
3570	static struct mlxsw_sp_nexthop *
3571	mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3572	struct mlxsw_sp_nexthop_key key)
3573	{
3574	return rhashtable_lookup_fast(ht: &mlxsw_sp->router->nexthop_ht, key: &key,
3575	params: mlxsw_sp_nexthop_ht_params);
3576	}
3577
3578	static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3579	enum mlxsw_sp_l3proto proto,
3580	u16 vr_id,
3581	u32 adj_index, u16 ecmp_size,
3582	u32 new_adj_index,
3583	u16 new_ecmp_size)
3584	{
3585	char raleu_pl[MLXSW_REG_RALEU_LEN];
3586
3587	mlxsw_reg_raleu_pack(payload: raleu_pl,
3588	protocol: (enum mlxsw_reg_ralxx_protocol) proto, virtual_router: vr_id,
3589	adjacency_index: adj_index, ecmp_size, new_adjacency_index: new_adj_index,
3590	new_ecmp_size);
3591	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(raleu), payload: raleu_pl);
3592	}
3593
3594	static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3595	struct mlxsw_sp_nexthop_group *nh_grp,
3596	u32 old_adj_index, u16 old_ecmp_size)
3597	{
3598	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3599	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3600	int err;
3601
3602	list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
3603	err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
3604	proto: vr_entry->key.proto,
3605	vr_id: vr_entry->key.vr_id,
3606	adj_index: old_adj_index,
3607	ecmp_size: old_ecmp_size,
3608	new_adj_index: nhgi->adj_index,
3609	new_ecmp_size: nhgi->ecmp_size);
3610	if (err)
3611	goto err_mass_update_vr;
3612	}
3613	return 0;
3614
3615	err_mass_update_vr:
3616	list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
3617	mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, proto: vr_entry->key.proto,
3618	vr_id: vr_entry->key.vr_id,
3619	adj_index: nhgi->adj_index,
3620	ecmp_size: nhgi->ecmp_size,
3621	new_adj_index: old_adj_index, new_ecmp_size: old_ecmp_size);
3622	return err;
3623	}
3624
3625	static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
3626	u32 adj_index,
3627	struct mlxsw_sp_nexthop *nh,
3628	bool force, char *ratr_pl)
3629	{
3630	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3631	struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh);
3632	enum mlxsw_reg_ratr_op op;
3633	u16 rif_index;
3634
3635	rif_index = rif ? rif->rif_index :
3636	mlxsw_sp->router->lb_crif->rif->rif_index;
3637	op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY :
3638	MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY;
3639	mlxsw_reg_ratr_pack(payload: ratr_pl, op, valid: true, type: MLXSW_REG_RATR_TYPE_ETHERNET,
3640	adjacency_index: adj_index, egress_rif: rif_index);
3641	switch (nh->action) {
3642	case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
3643	mlxsw_reg_ratr_eth_entry_pack(payload: ratr_pl, dest_mac: neigh_entry->ha);
3644	break;
3645	case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
3646	mlxsw_reg_ratr_trap_action_set(buf: ratr_pl,
3647	val: MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
3648	break;
3649	case MLXSW_SP_NEXTHOP_ACTION_TRAP:
3650	mlxsw_reg_ratr_trap_action_set(buf: ratr_pl,
3651	val: MLXSW_REG_RATR_TRAP_ACTION_TRAP);
3652	mlxsw_reg_ratr_trap_id_set(buf: ratr_pl, val: MLXSW_TRAP_ID_RTR_EGRESS0);
3653	break;
3654	default:
3655	WARN_ON_ONCE(1);
3656	return -EINVAL;
3657	}
3658	if (nh->counter_valid)
3659	mlxsw_reg_ratr_counter_pack(payload: ratr_pl, counter_index: nh->counter_index, counter_enable: true);
3660	else
3661	mlxsw_reg_ratr_counter_pack(payload: ratr_pl, counter_index: 0, counter_enable: false);
3662
3663	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ratr), payload: ratr_pl);
3664	}
3665
3666	int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3667	struct mlxsw_sp_nexthop *nh, bool force,
3668	char *ratr_pl)
3669	{
3670	int i;
3671
3672	for (i = 0; i < nh->num_adj_entries; i++) {
3673	int err;
3674
3675	err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index: adj_index + i,
3676	nh, force, ratr_pl);
3677	if (err)
3678	return err;
3679	}
3680
3681	return 0;
3682	}
3683
3684	static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3685	u32 adj_index,
3686	struct mlxsw_sp_nexthop *nh,
3687	bool force, char *ratr_pl)
3688	{
3689	const struct mlxsw_sp_ipip_ops *ipip_ops;
3690
3691	ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3692	return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry,
3693	force, ratr_pl);
3694	}
3695
3696	static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3697	u32 adj_index,
3698	struct mlxsw_sp_nexthop *nh, bool force,
3699	char *ratr_pl)
3700	{
3701	int i;
3702
3703	for (i = 0; i < nh->num_adj_entries; i++) {
3704	int err;
3705
3706	err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index: adj_index + i,
3707	nh, force, ratr_pl);
3708	if (err)
3709	return err;
3710	}
3711
3712	return 0;
3713	}
3714
3715	static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3716	struct mlxsw_sp_nexthop *nh, bool force,
3717	char *ratr_pl)
3718	{
3719	/* When action is discard or trap, the nexthop must be
3720	* programmed as an Ethernet nexthop.
3721	*/
3722	if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
3723	nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
3724	nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3725	return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh,
3726	force, ratr_pl);
3727	else
3728	return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh,
3729	force, ratr_pl);
3730	}
3731
3732	static int
3733	mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3734	struct mlxsw_sp_nexthop_group_info *nhgi,
3735	bool reallocate)
3736	{
3737	char ratr_pl[MLXSW_REG_RATR_LEN];
3738	u32 adj_index = nhgi->adj_index; /* base */
3739	struct mlxsw_sp_nexthop *nh;
3740	int i;
3741
3742	for (i = 0; i < nhgi->count; i++) {
3743	nh = &nhgi->nexthops[i];
3744
3745	if (!nh->should_offload) {
3746	nh->offloaded = 0;
3747	continue;
3748	}
3749
3750	if (nh->update || reallocate) {
3751	int err = 0;
3752
3753	err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh,
3754	force: true, ratr_pl);
3755	if (err)
3756	return err;
3757	nh->update = 0;
3758	nh->offloaded = 1;
3759	}
3760	adj_index += nh->num_adj_entries;
3761	}
3762	return 0;
3763	}
3764
3765	static int
3766	mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3767	struct mlxsw_sp_nexthop_group *nh_grp)
3768	{
3769	struct mlxsw_sp_fib_entry *fib_entry;
3770	int err;
3771
3772	list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3773	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3774	if (err)
3775	return err;
3776	}
3777	return 0;
3778	}
3779
3780	struct mlxsw_sp_adj_grp_size_range {
3781	u16 start; /* Inclusive */
3782	u16 end; /* Inclusive */
3783	};
3784
3785	/* Ordered by range start value */
3786	static const struct mlxsw_sp_adj_grp_size_range
3787	mlxsw_sp1_adj_grp_size_ranges[] = {
3788	{ .start = 1, .end = 64 },
3789	{ .start = 512, .end = 512 },
3790	{ .start = 1024, .end = 1024 },
3791	{ .start = 2048, .end = 2048 },
3792	{ .start = 4096, .end = 4096 },
3793	};
3794
3795	/* Ordered by range start value */
3796	static const struct mlxsw_sp_adj_grp_size_range
3797	mlxsw_sp2_adj_grp_size_ranges[] = {
3798	{ .start = 1, .end = 128 },
3799	{ .start = 256, .end = 256 },
3800	{ .start = 512, .end = 512 },
3801	{ .start = 1024, .end = 1024 },
3802	{ .start = 2048, .end = 2048 },
3803	{ .start = 4096, .end = 4096 },
3804	};
3805
3806	static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp,
3807	u16 *p_adj_grp_size)
3808	{
3809	int i;
3810
3811	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
3812	const struct mlxsw_sp_adj_grp_size_range *size_range;
3813
3814	size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3815
3816	if (*p_adj_grp_size >= size_range->start &&
3817	*p_adj_grp_size <= size_range->end)
3818	return;
3819
3820	if (*p_adj_grp_size <= size_range->end) {
3821	*p_adj_grp_size = size_range->end;
3822	return;
3823	}
3824	}
3825	}
3826
3827	static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp,
3828	u16 *p_adj_grp_size,
3829	unsigned int alloc_size)
3830	{
3831	int i;
3832
3833	for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) {
3834	const struct mlxsw_sp_adj_grp_size_range *size_range;
3835
3836	size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3837
3838	if (alloc_size >= size_range->end) {
3839	*p_adj_grp_size = size_range->end;
3840	return;
3841	}
3842	}
3843	}
3844
3845	static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3846	u16 *p_adj_grp_size)
3847	{
3848	unsigned int alloc_size;
3849	int err;
3850
3851	/* Round up the requested group size to the next size supported
3852	* by the device and make sure the request can be satisfied.
3853	*/
3854	mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size);
3855	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3856	type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3857	entry_count: *p_adj_grp_size, p_alloc_count: &alloc_size);
3858	if (err)
3859	return err;
3860	/* It is possible the allocation results in more allocated
3861	* entries than requested. Try to use as much of them as
3862	* possible.
3863	*/
3864	mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size);
3865
3866	return 0;
3867	}
3868
3869	static void
3870	mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3871	{
3872	int i, g = 0, sum_norm_weight = 0;
3873	struct mlxsw_sp_nexthop *nh;
3874
3875	for (i = 0; i < nhgi->count; i++) {
3876	nh = &nhgi->nexthops[i];
3877
3878	if (!nh->should_offload)
3879	continue;
3880	if (g > 0)
3881	g = gcd(a: nh->nh_weight, b: g);
3882	else
3883	g = nh->nh_weight;
3884	}
3885
3886	for (i = 0; i < nhgi->count; i++) {
3887	nh = &nhgi->nexthops[i];
3888
3889	if (!nh->should_offload)
3890	continue;
3891	nh->norm_nh_weight = nh->nh_weight / g;
3892	sum_norm_weight += nh->norm_nh_weight;
3893	}
3894
3895	nhgi->sum_norm_weight = sum_norm_weight;
3896	}
3897
3898	static void
3899	mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3900	{
3901	int i, weight = 0, lower_bound = 0;
3902	int total = nhgi->sum_norm_weight;
3903	u16 ecmp_size = nhgi->ecmp_size;
3904
3905	for (i = 0; i < nhgi->count; i++) {
3906	struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3907	int upper_bound;
3908
3909	if (!nh->should_offload)
3910	continue;
3911	weight += nh->norm_nh_weight;
3912	upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3913	nh->num_adj_entries = upper_bound - lower_bound;
3914	lower_bound = upper_bound;
3915	}
3916	}
3917
3918	static struct mlxsw_sp_nexthop *
3919	mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3920	const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3921
3922	static void
3923	mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3924	struct mlxsw_sp_nexthop_group *nh_grp)
3925	{
3926	int i;
3927
3928	for (i = 0; i < nh_grp->nhgi->count; i++) {
3929	struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3930
3931	if (nh->offloaded)
3932	nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3933	else
3934	nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3935	}
3936	}
3937
3938	static void
3939	__mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3940	struct mlxsw_sp_fib6_entry *fib6_entry)
3941	{
3942	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3943
3944	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3945	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3946	struct mlxsw_sp_nexthop *nh;
3947
3948	nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3949	if (nh && nh->offloaded)
3950	fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3951	else
3952	fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3953	}
3954	}
3955
3956	static void
3957	mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3958	struct mlxsw_sp_nexthop_group *nh_grp)
3959	{
3960	struct mlxsw_sp_fib6_entry *fib6_entry;
3961
3962	/* Unfortunately, in IPv6 the route and the nexthop are described by
3963	* the same struct, so we need to iterate over all the routes using the
3964	* nexthop group and set / clear the offload indication for them.
3965	*/
3966	list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3967	common.nexthop_group_node)
3968	__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3969	}
3970
3971	static void
3972	mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3973	const struct mlxsw_sp_nexthop *nh,
3974	u16 bucket_index)
3975	{
3976	struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp;
3977	bool offload = false, trap = false;
3978
3979	if (nh->offloaded) {
3980	if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3981	trap = true;
3982	else
3983	offload = true;
3984	}
3985	nexthop_bucket_set_hw_flags(net: mlxsw_sp_net(mlxsw_sp), id: nh_grp->obj.id,
3986	bucket_index, offload, trap);
3987	}
3988
3989	static void
3990	mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3991	struct mlxsw_sp_nexthop_group *nh_grp)
3992	{
3993	int i;
3994
3995	/* Do not update the flags if the nexthop group is being destroyed
3996	* since:
3997	* 1. The nexthop objects is being deleted, in which case the flags are
3998	* irrelevant.
3999	* 2. The nexthop group was replaced by a newer group, in which case
4000	* the flags of the nexthop object were already updated based on the
4001	* new group.
4002	*/
4003	if (nh_grp->can_destroy)
4004	return;
4005
4006	nexthop_set_hw_flags(net: mlxsw_sp_net(mlxsw_sp), id: nh_grp->obj.id,
4007	offload: nh_grp->nhgi->adj_index_valid, trap: false);
4008
4009	/* Update flags of individual nexthop buckets in case of a resilient
4010	* nexthop group.
4011	*/
4012	if (!nh_grp->nhgi->is_resilient)
4013	return;
4014
4015	for (i = 0; i < nh_grp->nhgi->count; i++) {
4016	struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4017
4018	mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index: i);
4019	}
4020	}
4021
4022	static void
4023	mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
4024	struct mlxsw_sp_nexthop_group *nh_grp)
4025	{
4026	switch (nh_grp->type) {
4027	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
4028	mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
4029	break;
4030	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
4031	mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
4032	break;
4033	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
4034	mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
4035	break;
4036	}
4037	}
4038
4039	static int
4040	mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
4041	struct mlxsw_sp_nexthop_group *nh_grp)
4042	{
4043	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4044	u16 ecmp_size, old_ecmp_size;
4045	struct mlxsw_sp_nexthop *nh;
4046	bool offload_change = false;
4047	u32 adj_index;
4048	bool old_adj_index_valid;
4049	u32 old_adj_index;
4050	int i, err2, err;
4051
4052	if (!nhgi->gateway)
4053	return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4054
4055	for (i = 0; i < nhgi->count; i++) {
4056	nh = &nhgi->nexthops[i];
4057
4058	if (nh->should_offload != nh->offloaded) {
4059	offload_change = true;
4060	if (nh->should_offload)
4061	nh->update = 1;
4062	}
4063	}
4064	if (!offload_change) {
4065	/* Nothing was added or removed, so no need to reallocate. Just
4066	* update MAC on existing adjacency indexes.
4067	*/
4068	err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, reallocate: false);
4069	if (err) {
4070	dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
4071	goto set_trap;
4072	}
4073	/* Flags of individual nexthop buckets might need to be
4074	* updated.
4075	*/
4076	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4077	return 0;
4078	}
4079	mlxsw_sp_nexthop_group_normalize(nhgi);
4080	if (!nhgi->sum_norm_weight) {
4081	/* No neigh of this group is connected so we just set
4082	* the trap and let everthing flow through kernel.
4083	*/
4084	err = 0;
4085	goto set_trap;
4086	}
4087
4088	ecmp_size = nhgi->sum_norm_weight;
4089	err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, p_adj_grp_size: &ecmp_size);
4090	if (err)
4091	/* No valid allocation size available. */
4092	goto set_trap;
4093
4094	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4095	entry_count: ecmp_size, p_entry_index: &adj_index);
4096	if (err) {
4097	/* We ran out of KVD linear space, just set the
4098	* trap and let everything flow through kernel.
4099	*/
4100	dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
4101	goto set_trap;
4102	}
4103	old_adj_index_valid = nhgi->adj_index_valid;
4104	old_adj_index = nhgi->adj_index;
4105	old_ecmp_size = nhgi->ecmp_size;
4106	nhgi->adj_index_valid = 1;
4107	nhgi->adj_index = adj_index;
4108	nhgi->ecmp_size = ecmp_size;
4109	mlxsw_sp_nexthop_group_rebalance(nhgi);
4110	err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, reallocate: true);
4111	if (err) {
4112	dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
4113	goto set_trap;
4114	}
4115
4116	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4117
4118	if (!old_adj_index_valid) {
4119	/* The trap was set for fib entries, so we have to call
4120	* fib entry update to unset it and use adjacency index.
4121	*/
4122	err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4123	if (err) {
4124	dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
4125	goto set_trap;
4126	}
4127	return 0;
4128	}
4129
4130	err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
4131	old_adj_index, old_ecmp_size);
4132	mlxsw_sp_kvdl_free(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4133	entry_count: old_ecmp_size, entry_index: old_adj_index);
4134	if (err) {
4135	dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
4136	goto set_trap;
4137	}
4138
4139	return 0;
4140
4141	set_trap:
4142	old_adj_index_valid = nhgi->adj_index_valid;
4143	nhgi->adj_index_valid = 0;
4144	for (i = 0; i < nhgi->count; i++) {
4145	nh = &nhgi->nexthops[i];
4146	nh->offloaded = 0;
4147	}
4148	err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4149	if (err2)
4150	dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
4151	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4152	if (old_adj_index_valid)
4153	mlxsw_sp_kvdl_free(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4154	entry_count: nhgi->ecmp_size, entry_index: nhgi->adj_index);
4155	return err;
4156	}
4157
4158	static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
4159	bool removing)
4160	{
4161	if (!removing) {
4162	nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
4163	nh->should_offload = 1;
4164	} else if (nh->nhgi->is_resilient) {
4165	nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
4166	nh->should_offload = 1;
4167	} else {
4168	nh->should_offload = 0;
4169	}
4170	nh->update = 1;
4171	}
4172
4173	static int
4174	mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
4175	struct mlxsw_sp_neigh_entry *neigh_entry)
4176	{
4177	struct neighbour *n, *old_n = neigh_entry->key.n;
4178	struct mlxsw_sp_nexthop *nh;
4179	struct net_device *dev;
4180	bool entry_connected;
4181	u8 nud_state, dead;
4182	int err;
4183
4184	nh = list_first_entry(&neigh_entry->nexthop_list,
4185	struct mlxsw_sp_nexthop, neigh_list_node);
4186	dev = mlxsw_sp_nexthop_dev(nh);
4187
4188	n = neigh_lookup(tbl: nh->neigh_tbl, pkey: &nh->gw_addr, dev);
4189	if (!n) {
4190	n = neigh_create(tbl: nh->neigh_tbl, pkey: &nh->gw_addr, dev);
4191	if (IS_ERR(ptr: n))
4192	return PTR_ERR(ptr: n);
4193	neigh_event_send(neigh: n, NULL);
4194	}
4195
4196	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
4197	neigh_entry->key.n = n;
4198	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4199	if (err)
4200	goto err_neigh_entry_insert;
4201
4202	read_lock_bh(&n->lock);
4203	nud_state = n->nud_state;
4204	dead = n->dead;
4205	read_unlock_bh(&n->lock);
4206	entry_connected = nud_state & NUD_VALID && !dead;
4207
4208	list_for_each_entry(nh, &neigh_entry->nexthop_list,
4209	neigh_list_node) {
4210	neigh_release(neigh: old_n);
4211	neigh_clone(neigh: n);
4212	__mlxsw_sp_nexthop_neigh_update(nh, removing: !entry_connected);
4213	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4214	}
4215
4216	neigh_release(neigh: n);
4217
4218	return 0;
4219
4220	err_neigh_entry_insert:
4221	neigh_entry->key.n = old_n;
4222	mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4223	neigh_release(neigh: n);
4224	return err;
4225	}
4226
4227	static void
4228	mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
4229	struct mlxsw_sp_neigh_entry *neigh_entry,
4230	bool removing, bool dead)
4231	{
4232	struct mlxsw_sp_nexthop *nh;
4233
4234	if (list_empty(head: &neigh_entry->nexthop_list))
4235	return;
4236
4237	if (dead) {
4238	int err;
4239
4240	err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
4241	neigh_entry);
4242	if (err)
4243	dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
4244	return;
4245	}
4246
4247	list_for_each_entry(nh, &neigh_entry->nexthop_list,
4248	neigh_list_node) {
4249	__mlxsw_sp_nexthop_neigh_update(nh, removing);
4250	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4251	}
4252	}
4253
4254	static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh,
4255	struct mlxsw_sp_crif *crif)
4256	{
4257	if (nh->crif)
4258	return;
4259
4260	nh->crif = crif;
4261	list_add(new: &nh->crif_list_node, head: &crif->nexthop_list);
4262	}
4263
4264	static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh)
4265	{
4266	if (!nh->crif)
4267	return;
4268
4269	list_del(entry: &nh->crif_list_node);
4270	nh->crif = NULL;
4271	}
4272
4273	static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
4274	struct mlxsw_sp_nexthop *nh)
4275	{
4276	struct mlxsw_sp_neigh_entry *neigh_entry;
4277	struct net_device *dev;
4278	struct neighbour *n;
4279	u8 nud_state, dead;
4280	int err;
4281
4282	if (WARN_ON(!nh->crif->rif))
4283	return 0;
4284
4285	if (!nh->nhgi->gateway || nh->neigh_entry)
4286	return 0;
4287	dev = mlxsw_sp_nexthop_dev(nh);
4288
4289	/* Take a reference of neigh here ensuring that neigh would
4290	* not be destructed before the nexthop entry is finished.
4291	* The reference is taken either in neigh_lookup() or
4292	* in neigh_create() in case n is not found.
4293	*/
4294	n = neigh_lookup(tbl: nh->neigh_tbl, pkey: &nh->gw_addr, dev);
4295	if (!n) {
4296	n = neigh_create(tbl: nh->neigh_tbl, pkey: &nh->gw_addr, dev);
4297	if (IS_ERR(ptr: n))
4298	return PTR_ERR(ptr: n);
4299	neigh_event_send(neigh: n, NULL);
4300	}
4301	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
4302	if (!neigh_entry) {
4303	neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
4304	if (IS_ERR(ptr: neigh_entry)) {
4305	err = -EINVAL;
4306	goto err_neigh_entry_create;
4307	}
4308	}
4309
4310	/* If that is the first nexthop connected to that neigh, add to
4311	* nexthop_neighs_list
4312	*/
4313	if (list_empty(head: &neigh_entry->nexthop_list))
4314	list_add_tail(new: &neigh_entry->nexthop_neighs_list_node,
4315	head: &mlxsw_sp->router->nexthop_neighs_list);
4316
4317	nh->neigh_entry = neigh_entry;
4318	list_add_tail(new: &nh->neigh_list_node, head: &neigh_entry->nexthop_list);
4319	read_lock_bh(&n->lock);
4320	nud_state = n->nud_state;
4321	dead = n->dead;
4322	read_unlock_bh(&n->lock);
4323	__mlxsw_sp_nexthop_neigh_update(nh, removing: !(nud_state & NUD_VALID && !dead));
4324
4325	return 0;
4326
4327	err_neigh_entry_create:
4328	neigh_release(neigh: n);
4329	return err;
4330	}
4331
4332	static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
4333	struct mlxsw_sp_nexthop *nh)
4334	{
4335	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
4336	struct neighbour *n;
4337
4338	if (!neigh_entry)
4339	return;
4340	n = neigh_entry->key.n;
4341
4342	__mlxsw_sp_nexthop_neigh_update(nh, removing: true);
4343	list_del(entry: &nh->neigh_list_node);
4344	nh->neigh_entry = NULL;
4345
4346	/* If that is the last nexthop connected to that neigh, remove from
4347	* nexthop_neighs_list
4348	*/
4349	if (list_empty(head: &neigh_entry->nexthop_list))
4350	list_del(entry: &neigh_entry->nexthop_neighs_list_node);
4351
4352	if (!neigh_entry->connected && list_empty(head: &neigh_entry->nexthop_list))
4353	mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4354
4355	neigh_release(neigh: n);
4356	}
4357
4358	static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
4359	{
4360	struct net_device *ul_dev;
4361	bool is_up;
4362
4363	rcu_read_lock();
4364	ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
4365	is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
4366	rcu_read_unlock();
4367
4368	return is_up;
4369	}
4370
4371	static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
4372	struct mlxsw_sp_nexthop *nh,
4373	struct mlxsw_sp_ipip_entry *ipip_entry)
4374	{
4375	struct mlxsw_sp_crif *crif;
4376	bool removing;
4377
4378	if (!nh->nhgi->gateway || nh->ipip_entry)
4379	return;
4380
4381	crif = mlxsw_sp_crif_lookup(router: mlxsw_sp->router, dev: ipip_entry->ol_dev);
4382	if (WARN_ON(!crif))
4383	return;
4384
4385	nh->ipip_entry = ipip_entry;
4386	removing = !mlxsw_sp_ipip_netdev_ul_up(ol_dev: ipip_entry->ol_dev);
4387	__mlxsw_sp_nexthop_neigh_update(nh, removing);
4388	mlxsw_sp_nexthop_crif_init(nh, crif);
4389	}
4390
4391	static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
4392	struct mlxsw_sp_nexthop *nh)
4393	{
4394	struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
4395
4396	if (!ipip_entry)
4397	return;
4398
4399	__mlxsw_sp_nexthop_neigh_update(nh, removing: true);
4400	nh->ipip_entry = NULL;
4401	}
4402
4403	static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4404	const struct fib_nh *fib_nh,
4405	enum mlxsw_sp_ipip_type *p_ipipt)
4406	{
4407	struct net_device *dev = fib_nh->fib_nh_dev;
4408
4409	return dev &&
4410	fib_nh->nh_parent->fib_type == RTN_UNICAST &&
4411	mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_type: p_ipipt);
4412	}
4413
4414	static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
4415	struct mlxsw_sp_nexthop *nh,
4416	const struct net_device *dev)
4417	{
4418	const struct mlxsw_sp_ipip_ops *ipip_ops;
4419	struct mlxsw_sp_ipip_entry *ipip_entry;
4420	struct mlxsw_sp_crif *crif;
4421	int err;
4422
4423	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev: dev);
4424	if (ipip_entry) {
4425	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4426	if (ipip_ops->can_offload(mlxsw_sp, dev)) {
4427	nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4428	mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4429	return 0;
4430	}
4431	}
4432
4433	nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4434	crif = mlxsw_sp_crif_lookup(router: mlxsw_sp->router, dev);
4435	if (!crif)
4436	return 0;
4437
4438	mlxsw_sp_nexthop_crif_init(nh, crif);
4439
4440	if (!crif->rif)
4441	return 0;
4442
4443	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4444	if (err)
4445	goto err_neigh_init;
4446
4447	return 0;
4448
4449	err_neigh_init:
4450	mlxsw_sp_nexthop_crif_fini(nh);
4451	return err;
4452	}
4453
4454	static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp,
4455	struct mlxsw_sp_nexthop *nh)
4456	{
4457	switch (nh->type) {
4458	case MLXSW_SP_NEXTHOP_TYPE_ETH:
4459	return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4460	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4461	break;
4462	}
4463
4464	return 0;
4465	}
4466
4467	static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp,
4468	struct mlxsw_sp_nexthop *nh)
4469	{
4470	switch (nh->type) {
4471	case MLXSW_SP_NEXTHOP_TYPE_ETH:
4472	mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
4473	break;
4474	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4475	mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
4476	break;
4477	}
4478	}
4479
4480	static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
4481	struct mlxsw_sp_nexthop *nh)
4482	{
4483	mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4484	mlxsw_sp_nexthop_crif_fini(nh);
4485	}
4486
4487	static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
4488	struct mlxsw_sp_nexthop_group *nh_grp,
4489	struct mlxsw_sp_nexthop *nh,
4490	struct fib_nh *fib_nh)
4491	{
4492	struct net_device *dev = fib_nh->fib_nh_dev;
4493	struct in_device *in_dev;
4494	int err;
4495
4496	nh->nhgi = nh_grp->nhgi;
4497	nh->key.fib_nh = fib_nh;
4498	#ifdef CONFIG_IP_ROUTE_MULTIPATH
4499	nh->nh_weight = fib_nh->fib_nh_weight;
4500	#else
4501	nh->nh_weight = 1;
4502	#endif
4503	memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
4504	nh->neigh_tbl = &arp_tbl;
4505	err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
4506	if (err)
4507	return err;
4508
4509	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4510	list_add_tail(new: &nh->router_list_node, head: &mlxsw_sp->router->nexthop_list);
4511
4512	if (!dev)
4513	return 0;
4514	nh->ifindex = dev->ifindex;
4515
4516	rcu_read_lock();
4517	in_dev = __in_dev_get_rcu(dev);
4518	if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
4519	fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
4520	rcu_read_unlock();
4521	return 0;
4522	}
4523	rcu_read_unlock();
4524
4525	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4526	if (err)
4527	goto err_nexthop_neigh_init;
4528
4529	return 0;
4530
4531	err_nexthop_neigh_init:
4532	list_del(entry: &nh->router_list_node);
4533	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4534	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4535	return err;
4536	}
4537
4538	static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4539	struct mlxsw_sp_nexthop *nh)
4540	{
4541	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4542	list_del(entry: &nh->router_list_node);
4543	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4544	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4545	}
4546
4547	static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4548	unsigned long event, struct fib_nh *fib_nh)
4549	{
4550	struct mlxsw_sp_nexthop_key key;
4551	struct mlxsw_sp_nexthop *nh;
4552
4553	key.fib_nh = fib_nh;
4554	nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4555	if (!nh)
4556	return;
4557
4558	switch (event) {
4559	case FIB_EVENT_NH_ADD:
4560	mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev: fib_nh->fib_nh_dev);
4561	break;
4562	case FIB_EVENT_NH_DEL:
4563	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4564	break;
4565	}
4566
4567	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4568	}
4569
4570	static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4571	struct mlxsw_sp_rif *rif)
4572	{
4573	struct net_device *dev = mlxsw_sp_rif_dev(rif);
4574	struct mlxsw_sp_nexthop *nh;
4575	bool removing;
4576
4577	list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) {
4578	switch (nh->type) {
4579	case MLXSW_SP_NEXTHOP_TYPE_ETH:
4580	removing = false;
4581	break;
4582	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4583	removing = !mlxsw_sp_ipip_netdev_ul_up(ol_dev: dev);
4584	break;
4585	default:
4586	WARN_ON(1);
4587	continue;
4588	}
4589
4590	__mlxsw_sp_nexthop_neigh_update(nh, removing);
4591	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4592	}
4593	}
4594
4595	static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
4596	struct mlxsw_sp_rif *rif)
4597	{
4598	struct mlxsw_sp_nexthop *nh, *tmp;
4599	unsigned int n = 0;
4600	int err;
4601
4602	list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4603	crif_list_node) {
4604	err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh);
4605	if (err)
4606	goto err_nexthop_type_rif;
4607	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4608	n++;
4609	}
4610
4611	return 0;
4612
4613	err_nexthop_type_rif:
4614	list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4615	crif_list_node) {
4616	if (!n--)
4617	break;
4618	mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4619	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4620	}
4621	return err;
4622	}
4623
4624	static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4625	struct mlxsw_sp_rif *rif)
4626	{
4627	struct mlxsw_sp_nexthop *nh, *tmp;
4628
4629	list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4630	crif_list_node) {
4631	mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4632	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp: nh->nhgi->nh_grp);
4633	}
4634	}
4635
4636	static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp)
4637	{
4638	enum mlxsw_reg_ratr_trap_action trap_action;
4639	char ratr_pl[MLXSW_REG_RATR_LEN];
4640	int err;
4641
4642	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, entry_count: 1,
4643	p_entry_index: &mlxsw_sp->router->adj_trap_index);
4644	if (err)
4645	return err;
4646
4647	trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP;
4648	mlxsw_reg_ratr_pack(payload: ratr_pl, op: MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, valid: true,
4649	type: MLXSW_REG_RATR_TYPE_ETHERNET,
4650	adjacency_index: mlxsw_sp->router->adj_trap_index,
4651	egress_rif: mlxsw_sp->router->lb_crif->rif->rif_index);
4652	mlxsw_reg_ratr_trap_action_set(buf: ratr_pl, val: trap_action);
4653	mlxsw_reg_ratr_trap_id_set(buf: ratr_pl, val: MLXSW_TRAP_ID_RTR_EGRESS0);
4654	err = mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ratr), payload: ratr_pl);
4655	if (err)
4656	goto err_ratr_write;
4657
4658	return 0;
4659
4660	err_ratr_write:
4661	mlxsw_sp_kvdl_free(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, entry_count: 1,
4662	entry_index: mlxsw_sp->router->adj_trap_index);
4663	return err;
4664	}
4665
4666	static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp)
4667	{
4668	mlxsw_sp_kvdl_free(mlxsw_sp, type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, entry_count: 1,
4669	entry_index: mlxsw_sp->router->adj_trap_index);
4670	}
4671
4672	static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp)
4673	{
4674	int err;
4675
4676	if (refcount_inc_not_zero(r: &mlxsw_sp->router->num_groups))
4677	return 0;
4678
4679	err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp);
4680	if (err)
4681	return err;
4682
4683	refcount_set(r: &mlxsw_sp->router->num_groups, n: 1);
4684
4685	return 0;
4686	}
4687
4688	static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp)
4689	{
4690	if (!refcount_dec_and_test(r: &mlxsw_sp->router->num_groups))
4691	return;
4692
4693	mlxsw_sp_adj_trap_entry_fini(mlxsw_sp);
4694	}
4695
4696	static void
4697	mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp,
4698	const struct mlxsw_sp_nexthop_group *nh_grp,
4699	unsigned long *activity)
4700	{
4701	char *ratrad_pl;
4702	int i, err;
4703
4704	ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL);
4705	if (!ratrad_pl)
4706	return;
4707
4708	mlxsw_reg_ratrad_pack(payload: ratrad_pl, adjacency_index: nh_grp->nhgi->adj_index,
4709	ecmp_size: nh_grp->nhgi->count);
4710	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ratrad), payload: ratrad_pl);
4711	if (err)
4712	goto out;
4713
4714	for (i = 0; i < nh_grp->nhgi->count; i++) {
4715	if (!mlxsw_reg_ratrad_activity_vector_get(buf: ratrad_pl, index: i))
4716	continue;
4717	bitmap_set(map: activity, start: i, nbits: 1);
4718	}
4719
4720	out:
4721	kfree(objp: ratrad_pl);
4722	}
4723
4724	#define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */
4725
4726	static void
4727	mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp,
4728	const struct mlxsw_sp_nexthop_group *nh_grp)
4729	{
4730	unsigned long *activity;
4731
4732	activity = bitmap_zalloc(nbits: nh_grp->nhgi->count, GFP_KERNEL);
4733	if (!activity)
4734	return;
4735
4736	mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity);
4737	nexthop_res_grp_activity_update(net: mlxsw_sp_net(mlxsw_sp), id: nh_grp->obj.id,
4738	num_buckets: nh_grp->nhgi->count, activity);
4739
4740	bitmap_free(bitmap: activity);
4741	}
4742
4743	static void
4744	mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp)
4745	{
4746	unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL;
4747
4748	mlxsw_core_schedule_dw(dwork: &mlxsw_sp->router->nh_grp_activity_dw,
4749	delay: msecs_to_jiffies(m: interval));
4750	}
4751
4752	static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work)
4753	{
4754	struct mlxsw_sp_nexthop_group_info *nhgi;
4755	struct mlxsw_sp_router *router;
4756	bool reschedule = false;
4757
4758	router = container_of(work, struct mlxsw_sp_router,
4759	nh_grp_activity_dw.work);
4760
4761	mutex_lock(&router->lock);
4762
4763	list_for_each_entry(nhgi, &router->nh_res_grp_list, list) {
4764	mlxsw_sp_nh_grp_activity_update(mlxsw_sp: router->mlxsw_sp, nh_grp: nhgi->nh_grp);
4765	reschedule = true;
4766	}
4767
4768	mutex_unlock(lock: &router->lock);
4769
4770	if (!reschedule)
4771	return;
4772	mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp: router->mlxsw_sp);
4773	}
4774
4775	static int
4776	mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
4777	const struct nh_notifier_single_info *nh,
4778	struct netlink_ext_ack *extack)
4779	{
4780	int err = -EINVAL;
4781
4782	if (nh->is_fdb)
4783	NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
4784	else if (nh->has_encap)
4785	NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
4786	else
4787	err = 0;
4788
4789	return err;
4790	}
4791
4792	static int
4793	mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
4794	const struct nh_notifier_single_info *nh,
4795	struct netlink_ext_ack *extack)
4796	{
4797	int err;
4798
4799	err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
4800	if (err)
4801	return err;
4802
4803	/* Device only nexthops with an IPIP device are programmed as
4804	* encapsulating adjacency entries.
4805	*/
4806	if (!nh->gw_family && !nh->is_reject &&
4807	!mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev: nh->dev, NULL)) {
4808	NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
4809	return -EINVAL;
4810	}
4811
4812	return 0;
4813	}
4814
4815	static int
4816	mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
4817	const struct nh_notifier_grp_info *nh_grp,
4818	struct netlink_ext_ack *extack)
4819	{
4820	int i;
4821
4822	if (nh_grp->is_fdb) {
4823	NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
4824	return -EINVAL;
4825	}
4826
4827	for (i = 0; i < nh_grp->num_nh; i++) {
4828	const struct nh_notifier_single_info *nh;
4829	int err;
4830
4831	nh = &nh_grp->nh_entries[i].nh;
4832	err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4833	extack);
4834	if (err)
4835	return err;
4836	}
4837
4838	return 0;
4839	}
4840
4841	static int
4842	mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp,
4843	const struct nh_notifier_res_table_info *nh_res_table,
4844	struct netlink_ext_ack *extack)
4845	{
4846	unsigned int alloc_size;
4847	bool valid_size = false;
4848	int err, i;
4849
4850	if (nh_res_table->num_nh_buckets < 32) {
4851	NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32");
4852	return -EINVAL;
4853	}
4854
4855	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
4856	const struct mlxsw_sp_adj_grp_size_range *size_range;
4857
4858	size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
4859
4860	if (nh_res_table->num_nh_buckets >= size_range->start &&
4861	nh_res_table->num_nh_buckets <= size_range->end) {
4862	valid_size = true;
4863	break;
4864	}
4865	}
4866
4867	if (!valid_size) {
4868	NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets");
4869	return -EINVAL;
4870	}
4871
4872	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
4873	type: MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4874	entry_count: nh_res_table->num_nh_buckets,
4875	p_alloc_count: &alloc_size);
4876	if (err || nh_res_table->num_nh_buckets != alloc_size) {
4877	NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition");
4878	return -EINVAL;
4879	}
4880
4881	return 0;
4882	}
4883
4884	static int
4885	mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp,
4886	const struct nh_notifier_res_table_info *nh_res_table,
4887	struct netlink_ext_ack *extack)
4888	{
4889	int err;
4890	u16 i;
4891
4892	err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp,
4893	nh_res_table,
4894	extack);
4895	if (err)
4896	return err;
4897
4898	for (i = 0; i < nh_res_table->num_nh_buckets; i++) {
4899	const struct nh_notifier_single_info *nh;
4900	int err;
4901
4902	nh = &nh_res_table->nhs[i];
4903	err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4904	extack);
4905	if (err)
4906	return err;
4907	}
4908
4909	return 0;
4910	}
4911
4912	static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
4913	unsigned long event,
4914	struct nh_notifier_info *info)
4915	{
4916	struct nh_notifier_single_info *nh;
4917
4918	if (event != NEXTHOP_EVENT_REPLACE &&
4919	event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE &&
4920	event != NEXTHOP_EVENT_BUCKET_REPLACE)
4921	return 0;
4922
4923	switch (info->type) {
4924	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4925	return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh: info->nh,
4926	extack: info->extack);
4927	case NH_NOTIFIER_INFO_TYPE_GRP:
4928	return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
4929	nh_grp: info->nh_grp,
4930	extack: info->extack);
4931	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4932	return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp,
4933	nh_res_table: info->nh_res_table,
4934	extack: info->extack);
4935	case NH_NOTIFIER_INFO_TYPE_RES_BUCKET:
4936	nh = &info->nh_res_bucket->new_nh;
4937	return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4938	extack: info->extack);
4939	default:
4940	NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
4941	return -EOPNOTSUPP;
4942	}
4943	}
4944
4945	static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
4946	const struct nh_notifier_info *info)
4947	{
4948	const struct net_device *dev;
4949
4950	switch (info->type) {
4951	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4952	dev = info->nh->dev;
4953	return info->nh->gw_family || info->nh->is_reject ||
4954	mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
4955	case NH_NOTIFIER_INFO_TYPE_GRP:
4956	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4957	/* Already validated earlier. */
4958	return true;
4959	default:
4960	return false;
4961	}
4962	}
4963
4964	static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp,
4965	struct mlxsw_sp_nexthop *nh)
4966	{
4967	nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD;
4968	nh->should_offload = 1;
4969	/* While nexthops that discard packets do not forward packets
4970	* via an egress RIF, they still need to be programmed using a
4971	* valid RIF, so use the loopback RIF created during init.
4972	*/
4973	nh->crif = mlxsw_sp->router->lb_crif;
4974	}
4975
4976	static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp,
4977	struct mlxsw_sp_nexthop *nh)
4978	{
4979	nh->crif = NULL;
4980	nh->should_offload = 0;
4981	}
4982
4983	static int
4984	mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
4985	struct mlxsw_sp_nexthop_group *nh_grp,
4986	struct mlxsw_sp_nexthop *nh,
4987	struct nh_notifier_single_info *nh_obj, int weight)
4988	{
4989	struct net_device *dev = nh_obj->dev;
4990	int err;
4991
4992	nh->nhgi = nh_grp->nhgi;
4993	nh->nh_weight = weight;
4994
4995	switch (nh_obj->gw_family) {
4996	case AF_INET:
4997	memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
4998	nh->neigh_tbl = &arp_tbl;
4999	break;
5000	case AF_INET6:
5001	memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
5002	#if IS_ENABLED(CONFIG_IPV6)
5003	nh->neigh_tbl = &nd_tbl;
5004	#endif
5005	break;
5006	}
5007
5008	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5009	list_add_tail(new: &nh->router_list_node, head: &mlxsw_sp->router->nexthop_list);
5010	nh->ifindex = dev->ifindex;
5011
5012	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
5013	if (err)
5014	goto err_type_init;
5015
5016	if (nh_obj->is_reject)
5017	mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh);
5018
5019	/* In a resilient nexthop group, all the nexthops must be written to
5020	* the adjacency table. Even if they do not have a valid neighbour or
5021	* RIF.
5022	*/
5023	if (nh_grp->nhgi->is_resilient && !nh->should_offload) {
5024	nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
5025	nh->should_offload = 1;
5026	}
5027
5028	return 0;
5029
5030	err_type_init:
5031	list_del(entry: &nh->router_list_node);
5032	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5033	return err;
5034	}
5035
5036	static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp,
5037	struct mlxsw_sp_nexthop *nh)
5038	{
5039	if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD)
5040	mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh);
5041	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5042	list_del(entry: &nh->router_list_node);
5043	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5044	nh->should_offload = 0;
5045	}
5046
5047	static int
5048	mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp,
5049	struct mlxsw_sp_nexthop_group *nh_grp,
5050	struct nh_notifier_info *info)
5051	{
5052	struct mlxsw_sp_nexthop_group_info *nhgi;
5053	struct mlxsw_sp_nexthop *nh;
5054	bool is_resilient = false;
5055	unsigned int nhs;
5056	int err, i;
5057
5058	switch (info->type) {
5059	case NH_NOTIFIER_INFO_TYPE_SINGLE:
5060	nhs = 1;
5061	break;
5062	case NH_NOTIFIER_INFO_TYPE_GRP:
5063	nhs = info->nh_grp->num_nh;
5064	break;
5065	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
5066	nhs = info->nh_res_table->num_nh_buckets;
5067	is_resilient = true;
5068	break;
5069	default:
5070	return -EINVAL;
5071	}
5072
5073	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5074	if (!nhgi)
5075	return -ENOMEM;
5076	nh_grp->nhgi = nhgi;
5077	nhgi->nh_grp = nh_grp;
5078	nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info);
5079	nhgi->is_resilient = is_resilient;
5080	nhgi->count = nhs;
5081	for (i = 0; i < nhgi->count; i++) {
5082	struct nh_notifier_single_info *nh_obj;
5083	int weight;
5084
5085	nh = &nhgi->nexthops[i];
5086	switch (info->type) {
5087	case NH_NOTIFIER_INFO_TYPE_SINGLE:
5088	nh_obj = info->nh;
5089	weight = 1;
5090	break;
5091	case NH_NOTIFIER_INFO_TYPE_GRP:
5092	nh_obj = &info->nh_grp->nh_entries[i].nh;
5093	weight = info->nh_grp->nh_entries[i].weight;
5094	break;
5095	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
5096	nh_obj = &info->nh_res_table->nhs[i];
5097	weight = 1;
5098	break;
5099	default:
5100	err = -EINVAL;
5101	goto err_nexthop_obj_init;
5102	}
5103	err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj,
5104	weight);
5105	if (err)
5106	goto err_nexthop_obj_init;
5107	}
5108	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5109	if (err)
5110	goto err_group_inc;
5111	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5112	if (err) {
5113	NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device");
5114	goto err_group_refresh;
5115	}
5116
5117	/* Add resilient nexthop groups to a list so that the activity of their
5118	* nexthop buckets will be periodically queried and cleared.
5119	*/
5120	if (nhgi->is_resilient) {
5121	if (list_empty(head: &mlxsw_sp->router->nh_res_grp_list))
5122	mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp);
5123	list_add(new: &nhgi->list, head: &mlxsw_sp->router->nh_res_grp_list);
5124	}
5125
5126	return 0;
5127
5128	err_group_refresh:
5129	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5130	err_group_inc:
5131	i = nhgi->count;
5132	err_nexthop_obj_init:
5133	for (i--; i >= 0; i--) {
5134	nh = &nhgi->nexthops[i];
5135	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5136	}
5137	kfree(objp: nhgi);
5138	return err;
5139	}
5140
5141	static void
5142	mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5143	struct mlxsw_sp_nexthop_group *nh_grp)
5144	{
5145	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5146	struct mlxsw_sp_router *router = mlxsw_sp->router;
5147	int i;
5148
5149	if (nhgi->is_resilient) {
5150	list_del(entry: &nhgi->list);
5151	if (list_empty(head: &mlxsw_sp->router->nh_res_grp_list))
5152	cancel_delayed_work(dwork: &router->nh_grp_activity_dw);
5153	}
5154
5155	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5156	for (i = nhgi->count - 1; i >= 0; i--) {
5157	struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5158
5159	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5160	}
5161	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5162	WARN_ON_ONCE(nhgi->adj_index_valid);
5163	kfree(objp: nhgi);
5164	}
5165
5166	static struct mlxsw_sp_nexthop_group *
5167	mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp,
5168	struct nh_notifier_info *info)
5169	{
5170	struct mlxsw_sp_nexthop_group *nh_grp;
5171	int err;
5172
5173	nh_grp = kzalloc(size: sizeof(*nh_grp), GFP_KERNEL);
5174	if (!nh_grp)
5175	return ERR_PTR(error: -ENOMEM);
5176	INIT_LIST_HEAD(list: &nh_grp->vr_list);
5177	err = rhashtable_init(ht: &nh_grp->vr_ht,
5178	params: &mlxsw_sp_nexthop_group_vr_ht_params);
5179	if (err)
5180	goto err_nexthop_group_vr_ht_init;
5181	INIT_LIST_HEAD(list: &nh_grp->fib_list);
5182	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5183	nh_grp->obj.id = info->id;
5184
5185	err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info);
5186	if (err)
5187	goto err_nexthop_group_info_init;
5188
5189	nh_grp->can_destroy = false;
5190
5191	return nh_grp;
5192
5193	err_nexthop_group_info_init:
5194	rhashtable_destroy(ht: &nh_grp->vr_ht);
5195	err_nexthop_group_vr_ht_init:
5196	kfree(objp: nh_grp);
5197	return ERR_PTR(error: err);
5198	}
5199
5200	static void
5201	mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp,
5202	struct mlxsw_sp_nexthop_group *nh_grp)
5203	{
5204	if (!nh_grp->can_destroy)
5205	return;
5206	mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp);
5207	WARN_ON_ONCE(!list_empty(&nh_grp->fib_list));
5208	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5209	rhashtable_destroy(ht: &nh_grp->vr_ht);
5210	kfree(objp: nh_grp);
5211	}
5212
5213	static struct mlxsw_sp_nexthop_group *
5214	mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id)
5215	{
5216	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
5217
5218	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5219	cmp_arg.id = id;
5220	return rhashtable_lookup_fast(ht: &mlxsw_sp->router->nexthop_group_ht,
5221	key: &cmp_arg,
5222	params: mlxsw_sp_nexthop_group_ht_params);
5223	}
5224
5225	static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp,
5226	struct mlxsw_sp_nexthop_group *nh_grp)
5227	{
5228	return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5229	}
5230
5231	static int
5232	mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp,
5233	struct mlxsw_sp_nexthop_group *nh_grp,
5234	struct mlxsw_sp_nexthop_group *old_nh_grp,
5235	struct netlink_ext_ack *extack)
5236	{
5237	struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi;
5238	struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi;
5239	int err;
5240
5241	old_nh_grp->nhgi = new_nhgi;
5242	new_nhgi->nh_grp = old_nh_grp;
5243	nh_grp->nhgi = old_nhgi;
5244	old_nhgi->nh_grp = nh_grp;
5245
5246	if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5247	/* Both the old adjacency index and the new one are valid.
5248	* Routes are currently using the old one. Tell the device to
5249	* replace the old adjacency index with the new one.
5250	*/
5251	err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp: old_nh_grp,
5252	old_adj_index: old_nhgi->adj_index,
5253	old_ecmp_size: old_nhgi->ecmp_size);
5254	if (err) {
5255	NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one");
5256	goto err_out;
5257	}
5258	} else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) {
5259	/* The old adjacency index is valid, while the new one is not.
5260	* Iterate over all the routes using the group and change them
5261	* to trap packets to the CPU.
5262	*/
5263	err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp: old_nh_grp);
5264	if (err) {
5265	NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets");
5266	goto err_out;
5267	}
5268	} else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5269	/* The old adjacency index is invalid, while the new one is.
5270	* Iterate over all the routes using the group and change them
5271	* to forward packets using the new valid index.
5272	*/
5273	err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp: old_nh_grp);
5274	if (err) {
5275	NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets");
5276	goto err_out;
5277	}
5278	}
5279
5280	/* Make sure the flags are set / cleared based on the new nexthop group
5281	* information.
5282	*/
5283	mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp: old_nh_grp);
5284
5285	/* At this point 'nh_grp' is just a shell that is not used by anyone
5286	* and its nexthop group info is the old info that was just replaced
5287	* with the new one. Remove it.
5288	*/
5289	nh_grp->can_destroy = true;
5290	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5291
5292	return 0;
5293
5294	err_out:
5295	old_nhgi->nh_grp = old_nh_grp;
5296	nh_grp->nhgi = new_nhgi;
5297	new_nhgi->nh_grp = nh_grp;
5298	old_nh_grp->nhgi = old_nhgi;
5299	return err;
5300	}
5301
5302	static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp,
5303	struct nh_notifier_info *info)
5304	{
5305	struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp;
5306	struct netlink_ext_ack *extack = info->extack;
5307	int err;
5308
5309	nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info);
5310	if (IS_ERR(ptr: nh_grp))
5311	return PTR_ERR(ptr: nh_grp);
5312
5313	old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, id: info->id);
5314	if (!old_nh_grp)
5315	err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp);
5316	else
5317	err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp,
5318	old_nh_grp, extack);
5319
5320	if (err) {
5321	nh_grp->can_destroy = true;
5322	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5323	}
5324
5325	return err;
5326	}
5327
5328	static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp,
5329	struct nh_notifier_info *info)
5330	{
5331	struct mlxsw_sp_nexthop_group *nh_grp;
5332
5333	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, id: info->id);
5334	if (!nh_grp)
5335	return;
5336
5337	nh_grp->can_destroy = true;
5338	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5339
5340	/* If the group still has routes using it, then defer the delete
5341	* operation until the last route using it is deleted.
5342	*/
5343	if (!list_empty(head: &nh_grp->fib_list))
5344	return;
5345	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5346	}
5347
5348	static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp,
5349	u32 adj_index, char *ratr_pl)
5350	{
5351	MLXSW_REG_ZERO(ratr, ratr_pl);
5352	mlxsw_reg_ratr_op_set(buf: ratr_pl, val: MLXSW_REG_RATR_OP_QUERY_READ);
5353	mlxsw_reg_ratr_adjacency_index_low_set(buf: ratr_pl, val: adj_index);
5354	mlxsw_reg_ratr_adjacency_index_high_set(buf: ratr_pl, val: adj_index >> 16);
5355
5356	return mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ratr), payload: ratr_pl);
5357	}
5358
5359	static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new)
5360	{
5361	/* Clear the opcode and activity on both the old and new payload as
5362	* they are irrelevant for the comparison.
5363	*/
5364	mlxsw_reg_ratr_op_set(buf: ratr_pl, val: MLXSW_REG_RATR_OP_QUERY_READ);
5365	mlxsw_reg_ratr_a_set(buf: ratr_pl, val: 0);
5366	mlxsw_reg_ratr_op_set(buf: ratr_pl_new, val: MLXSW_REG_RATR_OP_QUERY_READ);
5367	mlxsw_reg_ratr_a_set(buf: ratr_pl_new, val: 0);
5368
5369	/* If the contents of the adjacency entry are consistent with the
5370	* replacement request, then replacement was successful.
5371	*/
5372	if (!memcmp(p: ratr_pl, q: ratr_pl_new, MLXSW_REG_RATR_LEN))
5373	return 0;
5374
5375	return -EINVAL;
5376	}
5377
5378	static int
5379	mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp,
5380	struct mlxsw_sp_nexthop *nh,
5381	struct nh_notifier_info *info)
5382	{
5383	u16 bucket_index = info->nh_res_bucket->bucket_index;
5384	struct netlink_ext_ack *extack = info->extack;
5385	bool force = info->nh_res_bucket->force;
5386	char ratr_pl_new[MLXSW_REG_RATR_LEN];
5387	char ratr_pl[MLXSW_REG_RATR_LEN];
5388	u32 adj_index;
5389	int err;
5390
5391	/* No point in trying an atomic replacement if the idle timer interval
5392	* is smaller than the interval in which we query and clear activity.
5393	*/
5394	if (!force && info->nh_res_bucket->idle_timer_ms <
5395	MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL)
5396	force = true;
5397
5398	adj_index = nh->nhgi->adj_index + bucket_index;
5399	err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl);
5400	if (err) {
5401	NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket");
5402	return err;
5403	}
5404
5405	if (!force) {
5406	err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index,
5407	ratr_pl: ratr_pl_new);
5408	if (err) {
5409	NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent");
5410	return err;
5411	}
5412
5413	err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new);
5414	if (err) {
5415	NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement");
5416	return err;
5417	}
5418	}
5419
5420	nh->update = 0;
5421	nh->offloaded = 1;
5422	mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index);
5423
5424	return 0;
5425	}
5426
5427	static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp,
5428	struct nh_notifier_info *info)
5429	{
5430	u16 bucket_index = info->nh_res_bucket->bucket_index;
5431	struct netlink_ext_ack *extack = info->extack;
5432	struct mlxsw_sp_nexthop_group_info *nhgi;
5433	struct nh_notifier_single_info *nh_obj;
5434	struct mlxsw_sp_nexthop_group *nh_grp;
5435	struct mlxsw_sp_nexthop *nh;
5436	int err;
5437
5438	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, id: info->id);
5439	if (!nh_grp) {
5440	NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found");
5441	return -EINVAL;
5442	}
5443
5444	nhgi = nh_grp->nhgi;
5445
5446	if (bucket_index >= nhgi->count) {
5447	NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range");
5448	return -EINVAL;
5449	}
5450
5451	nh = &nhgi->nexthops[bucket_index];
5452	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5453
5454	nh_obj = &info->nh_res_bucket->new_nh;
5455	err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, weight: 1);
5456	if (err) {
5457	NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement");
5458	goto err_nexthop_obj_init;
5459	}
5460
5461	err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info);
5462	if (err)
5463	goto err_nexthop_obj_bucket_adj_update;
5464
5465	return 0;
5466
5467	err_nexthop_obj_bucket_adj_update:
5468	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5469	err_nexthop_obj_init:
5470	nh_obj = &info->nh_res_bucket->old_nh;
5471	mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, weight: 1);
5472	/* The old adjacency entry was not overwritten */
5473	nh->update = 0;
5474	nh->offloaded = 1;
5475	return err;
5476	}
5477
5478	static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb,
5479	unsigned long event, void *ptr)
5480	{
5481	struct nh_notifier_info *info = ptr;
5482	struct mlxsw_sp_router *router;
5483	int err = 0;
5484
5485	router = container_of(nb, struct mlxsw_sp_router, nexthop_nb);
5486	err = mlxsw_sp_nexthop_obj_validate(mlxsw_sp: router->mlxsw_sp, event, info);
5487	if (err)
5488	goto out;
5489
5490	mutex_lock(&router->lock);
5491
5492	switch (event) {
5493	case NEXTHOP_EVENT_REPLACE:
5494	err = mlxsw_sp_nexthop_obj_new(mlxsw_sp: router->mlxsw_sp, info);
5495	break;
5496	case NEXTHOP_EVENT_DEL:
5497	mlxsw_sp_nexthop_obj_del(mlxsw_sp: router->mlxsw_sp, info);
5498	break;
5499	case NEXTHOP_EVENT_BUCKET_REPLACE:
5500	err = mlxsw_sp_nexthop_obj_bucket_replace(mlxsw_sp: router->mlxsw_sp,
5501	info);
5502	break;
5503	default:
5504	break;
5505	}
5506
5507	mutex_unlock(lock: &router->lock);
5508
5509	out:
5510	return notifier_from_errno(err);
5511	}
5512
5513	static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5514	struct fib_info *fi)
5515	{
5516	const struct fib_nh *nh = fib_info_nh(fi, nhsel: 0);
5517
5518	return nh->fib_nh_gw_family ||
5519	mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh: nh, NULL);
5520	}
5521
5522	static int
5523	mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
5524	struct mlxsw_sp_nexthop_group *nh_grp)
5525	{
5526	unsigned int nhs = fib_info_num_path(fi: nh_grp->ipv4.fi);
5527	struct mlxsw_sp_nexthop_group_info *nhgi;
5528	struct mlxsw_sp_nexthop *nh;
5529	int err, i;
5530
5531	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5532	if (!nhgi)
5533	return -ENOMEM;
5534	nh_grp->nhgi = nhgi;
5535	nhgi->nh_grp = nh_grp;
5536	nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi: nh_grp->ipv4.fi);
5537	nhgi->count = nhs;
5538	for (i = 0; i < nhgi->count; i++) {
5539	struct fib_nh *fib_nh;
5540
5541	nh = &nhgi->nexthops[i];
5542	fib_nh = fib_info_nh(fi: nh_grp->ipv4.fi, nhsel: i);
5543	err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
5544	if (err)
5545	goto err_nexthop4_init;
5546	}
5547	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5548	if (err)
5549	goto err_group_inc;
5550	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5551	if (err)
5552	goto err_group_refresh;
5553
5554	return 0;
5555
5556	err_group_refresh:
5557	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5558	err_group_inc:
5559	i = nhgi->count;
5560	err_nexthop4_init:
5561	for (i--; i >= 0; i--) {
5562	nh = &nhgi->nexthops[i];
5563	mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5564	}
5565	kfree(objp: nhgi);
5566	return err;
5567	}
5568
5569	static void
5570	mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5571	struct mlxsw_sp_nexthop_group *nh_grp)
5572	{
5573	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5574	int i;
5575
5576	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5577	for (i = nhgi->count - 1; i >= 0; i--) {
5578	struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5579
5580	mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5581	}
5582	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5583	WARN_ON_ONCE(nhgi->adj_index_valid);
5584	kfree(objp: nhgi);
5585	}
5586
5587	static struct mlxsw_sp_nexthop_group *
5588	mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
5589	{
5590	struct mlxsw_sp_nexthop_group *nh_grp;
5591	int err;
5592
5593	nh_grp = kzalloc(size: sizeof(*nh_grp), GFP_KERNEL);
5594	if (!nh_grp)
5595	return ERR_PTR(error: -ENOMEM);
5596	INIT_LIST_HEAD(list: &nh_grp->vr_list);
5597	err = rhashtable_init(ht: &nh_grp->vr_ht,
5598	params: &mlxsw_sp_nexthop_group_vr_ht_params);
5599	if (err)
5600	goto err_nexthop_group_vr_ht_init;
5601	INIT_LIST_HEAD(list: &nh_grp->fib_list);
5602	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
5603	nh_grp->ipv4.fi = fi;
5604	fib_info_hold(fi);
5605
5606	err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
5607	if (err)
5608	goto err_nexthop_group_info_init;
5609
5610	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5611	if (err)
5612	goto err_nexthop_group_insert;
5613
5614	nh_grp->can_destroy = true;
5615
5616	return nh_grp;
5617
5618	err_nexthop_group_insert:
5619	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5620	err_nexthop_group_info_init:
5621	fib_info_put(fi);
5622	rhashtable_destroy(ht: &nh_grp->vr_ht);
5623	err_nexthop_group_vr_ht_init:
5624	kfree(objp: nh_grp);
5625	return ERR_PTR(error: err);
5626	}
5627
5628	static void
5629	mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
5630	struct mlxsw_sp_nexthop_group *nh_grp)
5631	{
5632	if (!nh_grp->can_destroy)
5633	return;
5634	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5635	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5636	fib_info_put(fi: nh_grp->ipv4.fi);
5637	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5638	rhashtable_destroy(ht: &nh_grp->vr_ht);
5639	kfree(objp: nh_grp);
5640	}
5641
5642	static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
5643	struct mlxsw_sp_fib_entry *fib_entry,
5644	struct fib_info *fi)
5645	{
5646	struct mlxsw_sp_nexthop_group *nh_grp;
5647
5648	if (fi->nh) {
5649	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
5650	id: fi->nh->id);
5651	if (WARN_ON_ONCE(!nh_grp))
5652	return -EINVAL;
5653	goto out;
5654	}
5655
5656	nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
5657	if (!nh_grp) {
5658	nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
5659	if (IS_ERR(ptr: nh_grp))
5660	return PTR_ERR(ptr: nh_grp);
5661	}
5662	out:
5663	list_add_tail(new: &fib_entry->nexthop_group_node, head: &nh_grp->fib_list);
5664	fib_entry->nh_group = nh_grp;
5665	return 0;
5666	}
5667
5668	static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
5669	struct mlxsw_sp_fib_entry *fib_entry)
5670	{
5671	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5672
5673	list_del(entry: &fib_entry->nexthop_group_node);
5674	if (!list_empty(head: &nh_grp->fib_list))
5675	return;
5676
5677	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
5678	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5679	return;
5680	}
5681
5682	mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
5683	}
5684
5685	static bool
5686	mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5687	{
5688	struct mlxsw_sp_fib4_entry *fib4_entry;
5689
5690	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5691	common);
5692	return !fib4_entry->dscp;
5693	}
5694
5695	static bool
5696	mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5697	{
5698	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5699
5700	switch (fib_entry->fib_node->fib->proto) {
5701	case MLXSW_SP_L3_PROTO_IPV4:
5702	if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
5703	return false;
5704	break;
5705	case MLXSW_SP_L3_PROTO_IPV6:
5706	break;
5707	}
5708
5709	switch (fib_entry->type) {
5710	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
5711	return !!nh_group->nhgi->adj_index_valid;
5712	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
5713	return !!mlxsw_sp_nhgi_rif(nhgi: nh_group->nhgi);
5714	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
5715	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5716	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
5717	return true;
5718	default:
5719	return false;
5720	}
5721	}
5722
5723	static struct mlxsw_sp_nexthop *
5724	mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
5725	const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5726	{
5727	int i;
5728
5729	for (i = 0; i < nh_grp->nhgi->count; i++) {
5730	struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
5731	struct net_device *dev = mlxsw_sp_nexthop_dev(nh);
5732	struct fib6_info *rt = mlxsw_sp_rt6->rt;
5733
5734	if (dev && dev == rt->fib6_nh->fib_nh_dev &&
5735	ipv6_addr_equal(a1: (const struct in6_addr *) &nh->gw_addr,
5736	a2: &rt->fib6_nh->fib_nh_gw6))
5737	return nh;
5738	}
5739
5740	return NULL;
5741	}
5742
5743	static void
5744	mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5745	struct fib_entry_notifier_info *fen_info)
5746	{
5747	u32 *p_dst = (u32 *) &fen_info->dst;
5748	struct fib_rt_info fri;
5749
5750	fri.fi = fen_info->fi;
5751	fri.tb_id = fen_info->tb_id;
5752	fri.dst = cpu_to_be32(*p_dst);
5753	fri.dst_len = fen_info->dst_len;
5754	fri.dscp = fen_info->dscp;
5755	fri.type = fen_info->type;
5756	fri.offload = false;
5757	fri.trap = false;
5758	fri.offload_failed = true;
5759	fib_alias_hw_flags_set(net: mlxsw_sp_net(mlxsw_sp), fri: &fri);
5760	}
5761
5762	static void
5763	mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5764	struct mlxsw_sp_fib_entry *fib_entry)
5765	{
5766	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5767	int dst_len = fib_entry->fib_node->key.prefix_len;
5768	struct mlxsw_sp_fib4_entry *fib4_entry;
5769	struct fib_rt_info fri;
5770	bool should_offload;
5771
5772	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5773	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5774	common);
5775	fri.fi = fib4_entry->fi;
5776	fri.tb_id = fib4_entry->tb_id;
5777	fri.dst = cpu_to_be32(*p_dst);
5778	fri.dst_len = dst_len;
5779	fri.dscp = fib4_entry->dscp;
5780	fri.type = fib4_entry->type;
5781	fri.offload = should_offload;
5782	fri.trap = !should_offload;
5783	fri.offload_failed = false;
5784	fib_alias_hw_flags_set(net: mlxsw_sp_net(mlxsw_sp), fri: &fri);
5785	}
5786
5787	static void
5788	mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5789	struct mlxsw_sp_fib_entry *fib_entry)
5790	{
5791	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5792	int dst_len = fib_entry->fib_node->key.prefix_len;
5793	struct mlxsw_sp_fib4_entry *fib4_entry;
5794	struct fib_rt_info fri;
5795
5796	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5797	common);
5798	fri.fi = fib4_entry->fi;
5799	fri.tb_id = fib4_entry->tb_id;
5800	fri.dst = cpu_to_be32(*p_dst);
5801	fri.dst_len = dst_len;
5802	fri.dscp = fib4_entry->dscp;
5803	fri.type = fib4_entry->type;
5804	fri.offload = false;
5805	fri.trap = false;
5806	fri.offload_failed = false;
5807	fib_alias_hw_flags_set(net: mlxsw_sp_net(mlxsw_sp), fri: &fri);
5808	}
5809
5810	#if IS_ENABLED(CONFIG_IPV6)
5811	static void
5812	mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5813	struct fib6_info **rt_arr,
5814	unsigned int nrt6)
5815	{
5816	int i;
5817
5818	/* In IPv6 a multipath route is represented using multiple routes, so
5819	* we need to set the flags on all of them.
5820	*/
5821	for (i = 0; i < nrt6; i++)
5822	fib6_info_hw_flags_set(net: mlxsw_sp_net(mlxsw_sp), f6i: rt_arr[i],
5823	offload: false, trap: false, offload_failed: true);
5824	}
5825	#else
5826	static void
5827	mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5828	struct fib6_info **rt_arr,
5829	unsigned int nrt6)
5830	{
5831	}
5832	#endif
5833
5834	#if IS_ENABLED(CONFIG_IPV6)
5835	static void
5836	mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5837	struct mlxsw_sp_fib_entry *fib_entry)
5838	{
5839	struct mlxsw_sp_fib6_entry *fib6_entry;
5840	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5841	bool should_offload;
5842
5843	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5844
5845	/* In IPv6 a multipath route is represented using multiple routes, so
5846	* we need to set the flags on all of them.
5847	*/
5848	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5849	common);
5850	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5851	fib6_info_hw_flags_set(net: mlxsw_sp_net(mlxsw_sp), f6i: mlxsw_sp_rt6->rt,
5852	offload: should_offload, trap: !should_offload, offload_failed: false);
5853	}
5854	#else
5855	static void
5856	mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5857	struct mlxsw_sp_fib_entry *fib_entry)
5858	{
5859	}
5860	#endif
5861
5862	#if IS_ENABLED(CONFIG_IPV6)
5863	static void
5864	mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5865	struct mlxsw_sp_fib_entry *fib_entry)
5866	{
5867	struct mlxsw_sp_fib6_entry *fib6_entry;
5868	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5869
5870	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5871	common);
5872	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5873	fib6_info_hw_flags_set(net: mlxsw_sp_net(mlxsw_sp), f6i: mlxsw_sp_rt6->rt,
5874	offload: false, trap: false, offload_failed: false);
5875	}
5876	#else
5877	static void
5878	mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5879	struct mlxsw_sp_fib_entry *fib_entry)
5880	{
5881	}
5882	#endif
5883
5884	static void
5885	mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5886	struct mlxsw_sp_fib_entry *fib_entry)
5887	{
5888	switch (fib_entry->fib_node->fib->proto) {
5889	case MLXSW_SP_L3_PROTO_IPV4:
5890	mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
5891	break;
5892	case MLXSW_SP_L3_PROTO_IPV6:
5893	mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
5894	break;
5895	}
5896	}
5897
5898	static void
5899	mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5900	struct mlxsw_sp_fib_entry *fib_entry)
5901	{
5902	switch (fib_entry->fib_node->fib->proto) {
5903	case MLXSW_SP_L3_PROTO_IPV4:
5904	mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5905	break;
5906	case MLXSW_SP_L3_PROTO_IPV6:
5907	mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5908	break;
5909	}
5910	}
5911
5912	static void
5913	mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
5914	struct mlxsw_sp_fib_entry *fib_entry,
5915	enum mlxsw_reg_ralue_op op)
5916	{
5917	switch (op) {
5918	case MLXSW_REG_RALUE_OP_WRITE_WRITE:
5919	mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
5920	break;
5921	case MLXSW_REG_RALUE_OP_WRITE_DELETE:
5922	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5923	break;
5924	default:
5925	break;
5926	}
5927	}
5928
5929	static void
5930	mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
5931	const struct mlxsw_sp_fib_entry *fib_entry,
5932	enum mlxsw_reg_ralue_op op)
5933	{
5934	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
5935	enum mlxsw_reg_ralxx_protocol proto;
5936	u32 *p_dip;
5937
5938	proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
5939
5940	switch (fib->proto) {
5941	case MLXSW_SP_L3_PROTO_IPV4:
5942	p_dip = (u32 *) fib_entry->fib_node->key.addr;
5943	mlxsw_reg_ralue_pack4(payload: ralue_pl, protocol: proto, op, virtual_router: fib->vr->id,
5944	prefix_len: fib_entry->fib_node->key.prefix_len,
5945	dip: *p_dip);
5946	break;
5947	case MLXSW_SP_L3_PROTO_IPV6:
5948	mlxsw_reg_ralue_pack6(payload: ralue_pl, protocol: proto, op, virtual_router: fib->vr->id,
5949	prefix_len: fib_entry->fib_node->key.prefix_len,
5950	dip: fib_entry->fib_node->key.addr);
5951	break;
5952	}
5953	}
5954
5955	static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
5956	struct mlxsw_sp_fib_entry *fib_entry,
5957	enum mlxsw_reg_ralue_op op)
5958	{
5959	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5960	struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
5961	char ralue_pl[MLXSW_REG_RALUE_LEN];
5962	enum mlxsw_reg_ralue_trap_action trap_action;
5963	u16 trap_id = 0;
5964	u32 adjacency_index = 0;
5965	u16 ecmp_size = 0;
5966
5967	/* In case the nexthop group adjacency index is valid, use it
5968	* with provided ECMP size. Otherwise, setup trap and pass
5969	* traffic to kernel.
5970	*/
5971	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5972	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5973	adjacency_index = nhgi->adj_index;
5974	ecmp_size = nhgi->ecmp_size;
5975	} else if (!nhgi->adj_index_valid && nhgi->count &&
5976	mlxsw_sp_nhgi_rif(nhgi)) {
5977	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5978	adjacency_index = mlxsw_sp->router->adj_trap_index;
5979	ecmp_size = 1;
5980	} else {
5981	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5982	trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5983	}
5984
5985	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5986	mlxsw_reg_ralue_act_remote_pack(payload: ralue_pl, trap_action, trap_id,
5987	adjacency_index, ecmp_size);
5988	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
5989	}
5990
5991	static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
5992	struct mlxsw_sp_fib_entry *fib_entry,
5993	enum mlxsw_reg_ralue_op op)
5994	{
5995	struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(nhgi: fib_entry->nh_group->nhgi);
5996	enum mlxsw_reg_ralue_trap_action trap_action;
5997	char ralue_pl[MLXSW_REG_RALUE_LEN];
5998	u16 trap_id = 0;
5999	u16 rif_index = 0;
6000
6001	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
6002	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
6003	rif_index = rif->rif_index;
6004	} else {
6005	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
6006	trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
6007	}
6008
6009	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6010	mlxsw_reg_ralue_act_local_pack(payload: ralue_pl, trap_action, trap_id,
6011	local_erif: rif_index);
6012	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
6013	}
6014
6015	static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
6016	struct mlxsw_sp_fib_entry *fib_entry,
6017	enum mlxsw_reg_ralue_op op)
6018	{
6019	char ralue_pl[MLXSW_REG_RALUE_LEN];
6020
6021	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6022	mlxsw_reg_ralue_act_ip2me_pack(payload: ralue_pl);
6023	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
6024	}
6025
6026	static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
6027	struct mlxsw_sp_fib_entry *fib_entry,
6028	enum mlxsw_reg_ralue_op op)
6029	{
6030	enum mlxsw_reg_ralue_trap_action trap_action;
6031	char ralue_pl[MLXSW_REG_RALUE_LEN];
6032
6033	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
6034	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6035	mlxsw_reg_ralue_act_local_pack(payload: ralue_pl, trap_action, trap_id: 0, local_erif: 0);
6036	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
6037	}
6038
6039	static int
6040	mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
6041	struct mlxsw_sp_fib_entry *fib_entry,
6042	enum mlxsw_reg_ralue_op op)
6043	{
6044	enum mlxsw_reg_ralue_trap_action trap_action;
6045	char ralue_pl[MLXSW_REG_RALUE_LEN];
6046	u16 trap_id;
6047
6048	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
6049	trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
6050
6051	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6052	mlxsw_reg_ralue_act_local_pack(payload: ralue_pl, trap_action, trap_id, local_erif: 0);
6053	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
6054	}
6055
6056	static int
6057	mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
6058	struct mlxsw_sp_fib_entry *fib_entry,
6059	enum mlxsw_reg_ralue_op op)
6060	{
6061	struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
6062	const struct mlxsw_sp_ipip_ops *ipip_ops;
6063	char ralue_pl[MLXSW_REG_RALUE_LEN];
6064	int err;
6065
6066	if (WARN_ON(!ipip_entry))
6067	return -EINVAL;
6068
6069	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
6070	err = ipip_ops->decap_config(mlxsw_sp, ipip_entry,
6071	fib_entry->decap.tunnel_index);
6072	if (err)
6073	return err;
6074
6075	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6076	mlxsw_reg_ralue_act_ip2me_tun_pack(payload: ralue_pl,
6077	tunnel_ptr: fib_entry->decap.tunnel_index);
6078	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
6079	}
6080
6081	static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
6082	struct mlxsw_sp_fib_entry *fib_entry,
6083	enum mlxsw_reg_ralue_op op)
6084	{
6085	char ralue_pl[MLXSW_REG_RALUE_LEN];
6086
6087	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6088	mlxsw_reg_ralue_act_ip2me_tun_pack(payload: ralue_pl,
6089	tunnel_ptr: fib_entry->decap.tunnel_index);
6090	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ralue), payload: ralue_pl);
6091	}
6092
6093	static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
6094	struct mlxsw_sp_fib_entry *fib_entry,
6095	enum mlxsw_reg_ralue_op op)
6096	{
6097	switch (fib_entry->type) {
6098	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
6099	return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
6100	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
6101	return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
6102	case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
6103	return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
6104	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
6105	return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op);
6106	case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
6107	return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry,
6108	op);
6109	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
6110	return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
6111	fib_entry, op);
6112	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
6113	return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op);
6114	}
6115	return -EINVAL;
6116	}
6117
6118	static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
6119	struct mlxsw_sp_fib_entry *fib_entry,
6120	enum mlxsw_reg_ralue_op op)
6121	{
6122	int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
6123
6124	if (err)
6125	return err;
6126
6127	mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
6128
6129	return err;
6130	}
6131
6132	static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
6133	struct mlxsw_sp_fib_entry *fib_entry)
6134	{
6135	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
6136	op: MLXSW_REG_RALUE_OP_WRITE_WRITE);
6137	}
6138
6139	static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
6140	struct mlxsw_sp_fib_entry *fib_entry)
6141	{
6142	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
6143	op: MLXSW_REG_RALUE_OP_WRITE_DELETE);
6144	}
6145
6146	static int
6147	mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
6148	const struct fib_entry_notifier_info *fen_info,
6149	struct mlxsw_sp_fib_entry *fib_entry)
6150	{
6151	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
6152	union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
6153	struct mlxsw_sp_router *router = mlxsw_sp->router;
6154	u32 tb_id = mlxsw_sp_fix_tb_id(tb_id: fen_info->tb_id);
6155	int ifindex = nhgi->nexthops[0].ifindex;
6156	struct mlxsw_sp_ipip_entry *ipip_entry;
6157
6158	switch (fen_info->type) {
6159	case RTN_LOCAL:
6160	ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ul_dev_ifindex: ifindex,
6161	ul_proto: MLXSW_SP_L3_PROTO_IPV4, ul_dip: dip);
6162	if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
6163	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
6164	return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
6165	fib_entry,
6166	ipip_entry);
6167	}
6168	if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, ul_tb_id: tb_id,
6169	ul_proto: MLXSW_SP_L3_PROTO_IPV4,
6170	ul_sip: &dip)) {
6171	u32 tunnel_index;
6172
6173	tunnel_index = router->nve_decap_config.tunnel_index;
6174	fib_entry->decap.tunnel_index = tunnel_index;
6175	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
6176	return 0;
6177	}
6178	fallthrough;
6179	case RTN_BROADCAST:
6180	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
6181	return 0;
6182	case RTN_BLACKHOLE:
6183	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
6184	return 0;
6185	case RTN_UNREACHABLE:
6186	case RTN_PROHIBIT:
6187	/* Packets hitting these routes need to be trapped, but
6188	* can do so with a lower priority than packets directed
6189	* at the host, so use action type local instead of trap.
6190	*/
6191	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
6192	return 0;
6193	case RTN_UNICAST:
6194	if (nhgi->gateway)
6195	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
6196	else
6197	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
6198	return 0;
6199	default:
6200	return -EINVAL;
6201	}
6202	}
6203
6204	static void
6205	mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6206	struct mlxsw_sp_fib_entry *fib_entry)
6207	{
6208	switch (fib_entry->type) {
6209	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
6210	mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
6211	break;
6212	default:
6213	break;
6214	}
6215	}
6216
6217	static void
6218	mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6219	struct mlxsw_sp_fib4_entry *fib4_entry)
6220	{
6221	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, fib_entry: &fib4_entry->common);
6222	}
6223
6224	static struct mlxsw_sp_fib4_entry *
6225	mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
6226	struct mlxsw_sp_fib_node *fib_node,
6227	const struct fib_entry_notifier_info *fen_info)
6228	{
6229	struct mlxsw_sp_fib4_entry *fib4_entry;
6230	struct mlxsw_sp_fib_entry *fib_entry;
6231	int err;
6232
6233	fib4_entry = kzalloc(size: sizeof(*fib4_entry), GFP_KERNEL);
6234	if (!fib4_entry)
6235	return ERR_PTR(error: -ENOMEM);
6236	fib_entry = &fib4_entry->common;
6237
6238	err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fi: fen_info->fi);
6239	if (err)
6240	goto err_nexthop4_group_get;
6241
6242	err = mlxsw_sp_nexthop_group_vr_link(nh_grp: fib_entry->nh_group,
6243	fib: fib_node->fib);
6244	if (err)
6245	goto err_nexthop_group_vr_link;
6246
6247	err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
6248	if (err)
6249	goto err_fib4_entry_type_set;
6250
6251	fib4_entry->fi = fen_info->fi;
6252	fib_info_hold(fi: fib4_entry->fi);
6253	fib4_entry->tb_id = fen_info->tb_id;
6254	fib4_entry->type = fen_info->type;
6255	fib4_entry->dscp = fen_info->dscp;
6256
6257	fib_entry->fib_node = fib_node;
6258
6259	return fib4_entry;
6260
6261	err_fib4_entry_type_set:
6262	mlxsw_sp_nexthop_group_vr_unlink(nh_grp: fib_entry->nh_group, fib: fib_node->fib);
6263	err_nexthop_group_vr_link:
6264	mlxsw_sp_nexthop4_group_put(mlxsw_sp, fib_entry: &fib4_entry->common);
6265	err_nexthop4_group_get:
6266	kfree(objp: fib4_entry);
6267	return ERR_PTR(error: err);
6268	}
6269
6270	static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
6271	struct mlxsw_sp_fib4_entry *fib4_entry)
6272	{
6273	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6274
6275	fib_info_put(fi: fib4_entry->fi);
6276	mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry);
6277	mlxsw_sp_nexthop_group_vr_unlink(nh_grp: fib4_entry->common.nh_group,
6278	fib: fib_node->fib);
6279	mlxsw_sp_nexthop4_group_put(mlxsw_sp, fib_entry: &fib4_entry->common);
6280	kfree(objp: fib4_entry);
6281	}
6282
6283	static struct mlxsw_sp_fib4_entry *
6284	mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
6285	const struct fib_entry_notifier_info *fen_info)
6286	{
6287	struct mlxsw_sp_fib4_entry *fib4_entry;
6288	struct mlxsw_sp_fib_node *fib_node;
6289	struct mlxsw_sp_fib *fib;
6290	struct mlxsw_sp_vr *vr;
6291
6292	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id: fen_info->tb_id);
6293	if (!vr)
6294	return NULL;
6295	fib = mlxsw_sp_vr_fib(vr, proto: MLXSW_SP_L3_PROTO_IPV4);
6296
6297	fib_node = mlxsw_sp_fib_node_lookup(fib, addr: &fen_info->dst,
6298	addr_len: sizeof(fen_info->dst),
6299	prefix_len: fen_info->dst_len);
6300	if (!fib_node)
6301	return NULL;
6302
6303	fib4_entry = container_of(fib_node->fib_entry,
6304	struct mlxsw_sp_fib4_entry, common);
6305	if (fib4_entry->tb_id == fen_info->tb_id &&
6306	fib4_entry->dscp == fen_info->dscp &&
6307	fib4_entry->type == fen_info->type &&
6308	fib4_entry->fi == fen_info->fi)
6309	return fib4_entry;
6310
6311	return NULL;
6312	}
6313
6314	static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
6315	.key_offset = offsetof(struct mlxsw_sp_fib_node, key),
6316	.head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
6317	.key_len = sizeof(struct mlxsw_sp_fib_key),
6318	.automatic_shrinking = true,
6319	};
6320
6321	static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
6322	struct mlxsw_sp_fib_node *fib_node)
6323	{
6324	return rhashtable_insert_fast(ht: &fib->ht, obj: &fib_node->ht_node,
6325	params: mlxsw_sp_fib_ht_params);
6326	}
6327
6328	static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
6329	struct mlxsw_sp_fib_node *fib_node)
6330	{
6331	rhashtable_remove_fast(ht: &fib->ht, obj: &fib_node->ht_node,
6332	params: mlxsw_sp_fib_ht_params);
6333	}
6334
6335	static struct mlxsw_sp_fib_node *
6336	mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
6337	size_t addr_len, unsigned char prefix_len)
6338	{
6339	struct mlxsw_sp_fib_key key;
6340
6341	memset(&key, 0, sizeof(key));
6342	memcpy(key.addr, addr, addr_len);
6343	key.prefix_len = prefix_len;
6344	return rhashtable_lookup_fast(ht: &fib->ht, key: &key, params: mlxsw_sp_fib_ht_params);
6345	}
6346
6347	static struct mlxsw_sp_fib_node *
6348	mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
6349	size_t addr_len, unsigned char prefix_len)
6350	{
6351	struct mlxsw_sp_fib_node *fib_node;
6352
6353	fib_node = kzalloc(size: sizeof(*fib_node), GFP_KERNEL);
6354	if (!fib_node)
6355	return NULL;
6356
6357	list_add(new: &fib_node->list, head: &fib->node_list);
6358	memcpy(fib_node->key.addr, addr, addr_len);
6359	fib_node->key.prefix_len = prefix_len;
6360
6361	return fib_node;
6362	}
6363
6364	static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
6365	{
6366	list_del(entry: &fib_node->list);
6367	kfree(objp: fib_node);
6368	}
6369
6370	static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
6371	struct mlxsw_sp_fib_node *fib_node)
6372	{
6373	struct mlxsw_sp_prefix_usage req_prefix_usage;
6374	struct mlxsw_sp_fib *fib = fib_node->fib;
6375	struct mlxsw_sp_lpm_tree *lpm_tree;
6376	int err;
6377
6378	lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
6379	if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6380	goto out;
6381
6382	mlxsw_sp_prefix_usage_cpy(prefix_usage1: &req_prefix_usage, prefix_usage2: &lpm_tree->prefix_usage);
6383	mlxsw_sp_prefix_usage_set(prefix_usage: &req_prefix_usage, prefix_len: fib_node->key.prefix_len);
6384	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, prefix_usage: &req_prefix_usage,
6385	proto: fib->proto);
6386	if (IS_ERR(ptr: lpm_tree))
6387	return PTR_ERR(ptr: lpm_tree);
6388
6389	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, new_tree: lpm_tree);
6390	if (err)
6391	goto err_lpm_tree_replace;
6392
6393	out:
6394	lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
6395	return 0;
6396
6397	err_lpm_tree_replace:
6398	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6399	return err;
6400	}
6401
6402	static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
6403	struct mlxsw_sp_fib_node *fib_node)
6404	{
6405	struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
6406	struct mlxsw_sp_prefix_usage req_prefix_usage;
6407	struct mlxsw_sp_fib *fib = fib_node->fib;
6408	int err;
6409
6410	if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6411	return;
6412	/* Try to construct a new LPM tree from the current prefix usage
6413	* minus the unused one. If we fail, continue using the old one.
6414	*/
6415	mlxsw_sp_prefix_usage_cpy(prefix_usage1: &req_prefix_usage, prefix_usage2: &lpm_tree->prefix_usage);
6416	mlxsw_sp_prefix_usage_clear(prefix_usage: &req_prefix_usage,
6417	prefix_len: fib_node->key.prefix_len);
6418	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, prefix_usage: &req_prefix_usage,
6419	proto: fib->proto);
6420	if (IS_ERR(ptr: lpm_tree))
6421	return;
6422
6423	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, new_tree: lpm_tree);
6424	if (err)
6425	goto err_lpm_tree_replace;
6426
6427	return;
6428
6429	err_lpm_tree_replace:
6430	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6431	}
6432
6433	static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
6434	struct mlxsw_sp_fib_node *fib_node,
6435	struct mlxsw_sp_fib *fib)
6436	{
6437	int err;
6438
6439	err = mlxsw_sp_fib_node_insert(fib, fib_node);
6440	if (err)
6441	return err;
6442	fib_node->fib = fib;
6443
6444	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
6445	if (err)
6446	goto err_fib_lpm_tree_link;
6447
6448	return 0;
6449
6450	err_fib_lpm_tree_link:
6451	fib_node->fib = NULL;
6452	mlxsw_sp_fib_node_remove(fib, fib_node);
6453	return err;
6454	}
6455
6456	static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
6457	struct mlxsw_sp_fib_node *fib_node)
6458	{
6459	struct mlxsw_sp_fib *fib = fib_node->fib;
6460
6461	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
6462	fib_node->fib = NULL;
6463	mlxsw_sp_fib_node_remove(fib, fib_node);
6464	}
6465
6466	static struct mlxsw_sp_fib_node *
6467	mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
6468	size_t addr_len, unsigned char prefix_len,
6469	enum mlxsw_sp_l3proto proto)
6470	{
6471	struct mlxsw_sp_fib_node *fib_node;
6472	struct mlxsw_sp_fib *fib;
6473	struct mlxsw_sp_vr *vr;
6474	int err;
6475
6476	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
6477	if (IS_ERR(ptr: vr))
6478	return ERR_CAST(ptr: vr);
6479	fib = mlxsw_sp_vr_fib(vr, proto);
6480
6481	fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
6482	if (fib_node)
6483	return fib_node;
6484
6485	fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
6486	if (!fib_node) {
6487	err = -ENOMEM;
6488	goto err_fib_node_create;
6489	}
6490
6491	err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
6492	if (err)
6493	goto err_fib_node_init;
6494
6495	return fib_node;
6496
6497	err_fib_node_init:
6498	mlxsw_sp_fib_node_destroy(fib_node);
6499	err_fib_node_create:
6500	mlxsw_sp_vr_put(mlxsw_sp, vr);
6501	return ERR_PTR(error: err);
6502	}
6503
6504	static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
6505	struct mlxsw_sp_fib_node *fib_node)
6506	{
6507	struct mlxsw_sp_vr *vr = fib_node->fib->vr;
6508
6509	if (fib_node->fib_entry)
6510	return;
6511	mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
6512	mlxsw_sp_fib_node_destroy(fib_node);
6513	mlxsw_sp_vr_put(mlxsw_sp, vr);
6514	}
6515
6516	static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
6517	struct mlxsw_sp_fib_entry *fib_entry)
6518	{
6519	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6520	int err;
6521
6522	fib_node->fib_entry = fib_entry;
6523
6524	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
6525	if (err)
6526	goto err_fib_entry_update;
6527
6528	return 0;
6529
6530	err_fib_entry_update:
6531	fib_node->fib_entry = NULL;
6532	return err;
6533	}
6534
6535	static void
6536	mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
6537	struct mlxsw_sp_fib_entry *fib_entry)
6538	{
6539	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6540
6541	mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
6542	fib_node->fib_entry = NULL;
6543	}
6544
6545	static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
6546	{
6547	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6548	struct mlxsw_sp_fib4_entry *fib4_replaced;
6549
6550	if (!fib_node->fib_entry)
6551	return true;
6552
6553	fib4_replaced = container_of(fib_node->fib_entry,
6554	struct mlxsw_sp_fib4_entry, common);
6555	if (fib4_entry->tb_id == RT_TABLE_MAIN &&
6556	fib4_replaced->tb_id == RT_TABLE_LOCAL)
6557	return false;
6558
6559	return true;
6560	}
6561
6562	static int
6563	mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
6564	const struct fib_entry_notifier_info *fen_info)
6565	{
6566	struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
6567	struct mlxsw_sp_fib_entry *replaced;
6568	struct mlxsw_sp_fib_node *fib_node;
6569	int err;
6570
6571	if (fen_info->fi->nh &&
6572	!mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, id: fen_info->fi->nh->id))
6573	return 0;
6574
6575	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, tb_id: fen_info->tb_id,
6576	addr: &fen_info->dst, addr_len: sizeof(fen_info->dst),
6577	prefix_len: fen_info->dst_len,
6578	proto: MLXSW_SP_L3_PROTO_IPV4);
6579	if (IS_ERR(ptr: fib_node)) {
6580	dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
6581	return PTR_ERR(ptr: fib_node);
6582	}
6583
6584	fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
6585	if (IS_ERR(ptr: fib4_entry)) {
6586	dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
6587	err = PTR_ERR(ptr: fib4_entry);
6588	goto err_fib4_entry_create;
6589	}
6590
6591	if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
6592	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6593	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6594	return 0;
6595	}
6596
6597	replaced = fib_node->fib_entry;
6598	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, fib_entry: &fib4_entry->common);
6599	if (err) {
6600	dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
6601	goto err_fib_node_entry_link;
6602	}
6603
6604	/* Nothing to replace */
6605	if (!replaced)
6606	return 0;
6607
6608	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry: replaced);
6609	fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
6610	common);
6611	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry: fib4_replaced);
6612
6613	return 0;
6614
6615	err_fib_node_entry_link:
6616	fib_node->fib_entry = replaced;
6617	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6618	err_fib4_entry_create:
6619	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6620	return err;
6621	}
6622
6623	static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
6624	struct fib_entry_notifier_info *fen_info)
6625	{
6626	struct mlxsw_sp_fib4_entry *fib4_entry;
6627	struct mlxsw_sp_fib_node *fib_node;
6628
6629	fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
6630	if (!fib4_entry)
6631	return;
6632	fib_node = fib4_entry->common.fib_node;
6633
6634	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_entry: &fib4_entry->common);
6635	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6636	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6637	}
6638
6639	static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
6640	{
6641	/* Multicast routes aren't supported, so ignore them. Neighbour
6642	* Discovery packets are specifically trapped.
6643	*/
6644	if (ipv6_addr_type(addr: &rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
6645	return true;
6646
6647	/* Cloned routes are irrelevant in the forwarding path. */
6648	if (rt->fib6_flags & RTF_CACHE)
6649	return true;
6650
6651	return false;
6652	}
6653
6654	static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
6655	{
6656	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6657
6658	mlxsw_sp_rt6 = kzalloc(size: sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
6659	if (!mlxsw_sp_rt6)
6660	return ERR_PTR(error: -ENOMEM);
6661
6662	/* In case of route replace, replaced route is deleted with
6663	* no notification. Take reference to prevent accessing freed
6664	* memory.
6665	*/
6666	mlxsw_sp_rt6->rt = rt;
6667	fib6_info_hold(f6i: rt);
6668
6669	return mlxsw_sp_rt6;
6670	}
6671
6672	#if IS_ENABLED(CONFIG_IPV6)
6673	static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6674	{
6675	fib6_info_release(f6i: rt);
6676	}
6677	#else
6678	static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6679	{
6680	}
6681	#endif
6682
6683	static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
6684	{
6685	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
6686
6687	if (!mlxsw_sp_rt6->rt->nh)
6688	fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
6689	mlxsw_sp_rt6_release(rt: mlxsw_sp_rt6->rt);
6690	kfree(objp: mlxsw_sp_rt6);
6691	}
6692
6693	static struct fib6_info *
6694	mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
6695	{
6696	return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
6697	list)->rt;
6698	}
6699
6700	static struct mlxsw_sp_rt6 *
6701	mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
6702	const struct fib6_info *rt)
6703	{
6704	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6705
6706	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
6707	if (mlxsw_sp_rt6->rt == rt)
6708	return mlxsw_sp_rt6;
6709	}
6710
6711	return NULL;
6712	}
6713
6714	static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
6715	const struct fib6_info *rt,
6716	enum mlxsw_sp_ipip_type *ret)
6717	{
6718	return rt->fib6_nh->fib_nh_dev &&
6719	mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev: rt->fib6_nh->fib_nh_dev, p_type: ret);
6720	}
6721
6722	static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
6723	struct mlxsw_sp_nexthop_group *nh_grp,
6724	struct mlxsw_sp_nexthop *nh,
6725	const struct fib6_info *rt)
6726	{
6727	struct net_device *dev = rt->fib6_nh->fib_nh_dev;
6728	int err;
6729
6730	nh->nhgi = nh_grp->nhgi;
6731	nh->nh_weight = rt->fib6_nh->fib_nh_weight;
6732	memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
6733	#if IS_ENABLED(CONFIG_IPV6)
6734	nh->neigh_tbl = &nd_tbl;
6735	#endif
6736	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
6737
6738	list_add_tail(new: &nh->router_list_node, head: &mlxsw_sp->router->nexthop_list);
6739
6740	if (!dev)
6741	return 0;
6742	nh->ifindex = dev->ifindex;
6743
6744	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
6745	if (err)
6746	goto err_nexthop_type_init;
6747
6748	return 0;
6749
6750	err_nexthop_type_init:
6751	list_del(entry: &nh->router_list_node);
6752	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6753	return err;
6754	}
6755
6756	static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
6757	struct mlxsw_sp_nexthop *nh)
6758	{
6759	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
6760	list_del(entry: &nh->router_list_node);
6761	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6762	}
6763
6764	static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
6765	const struct fib6_info *rt)
6766	{
6767	return rt->fib6_nh->fib_nh_gw_family ||
6768	mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
6769	}
6770
6771	static int
6772	mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
6773	struct mlxsw_sp_nexthop_group *nh_grp,
6774	struct mlxsw_sp_fib6_entry *fib6_entry)
6775	{
6776	struct mlxsw_sp_nexthop_group_info *nhgi;
6777	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6778	struct mlxsw_sp_nexthop *nh;
6779	int err, i;
6780
6781	nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
6782	GFP_KERNEL);
6783	if (!nhgi)
6784	return -ENOMEM;
6785	nh_grp->nhgi = nhgi;
6786	nhgi->nh_grp = nh_grp;
6787	mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
6788	struct mlxsw_sp_rt6, list);
6789	nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt: mlxsw_sp_rt6->rt);
6790	nhgi->count = fib6_entry->nrt6;
6791	for (i = 0; i < nhgi->count; i++) {
6792	struct fib6_info *rt = mlxsw_sp_rt6->rt;
6793
6794	nh = &nhgi->nexthops[i];
6795	err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
6796	if (err)
6797	goto err_nexthop6_init;
6798	mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
6799	}
6800	nh_grp->nhgi = nhgi;
6801	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
6802	if (err)
6803	goto err_group_inc;
6804	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6805	if (err)
6806	goto err_group_refresh;
6807
6808	return 0;
6809
6810	err_group_refresh:
6811	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6812	err_group_inc:
6813	i = nhgi->count;
6814	err_nexthop6_init:
6815	for (i--; i >= 0; i--) {
6816	nh = &nhgi->nexthops[i];
6817	mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6818	}
6819	kfree(objp: nhgi);
6820	return err;
6821	}
6822
6823	static void
6824	mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
6825	struct mlxsw_sp_nexthop_group *nh_grp)
6826	{
6827	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
6828	int i;
6829
6830	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6831	for (i = nhgi->count - 1; i >= 0; i--) {
6832	struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
6833
6834	mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6835	}
6836	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6837	WARN_ON_ONCE(nhgi->adj_index_valid);
6838	kfree(objp: nhgi);
6839	}
6840
6841	static struct mlxsw_sp_nexthop_group *
6842	mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
6843	struct mlxsw_sp_fib6_entry *fib6_entry)
6844	{
6845	struct mlxsw_sp_nexthop_group *nh_grp;
6846	int err;
6847
6848	nh_grp = kzalloc(size: sizeof(*nh_grp), GFP_KERNEL);
6849	if (!nh_grp)
6850	return ERR_PTR(error: -ENOMEM);
6851	INIT_LIST_HEAD(list: &nh_grp->vr_list);
6852	err = rhashtable_init(ht: &nh_grp->vr_ht,
6853	params: &mlxsw_sp_nexthop_group_vr_ht_params);
6854	if (err)
6855	goto err_nexthop_group_vr_ht_init;
6856	INIT_LIST_HEAD(list: &nh_grp->fib_list);
6857	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
6858
6859	err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
6860	if (err)
6861	goto err_nexthop_group_info_init;
6862
6863	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
6864	if (err)
6865	goto err_nexthop_group_insert;
6866
6867	nh_grp->can_destroy = true;
6868
6869	return nh_grp;
6870
6871	err_nexthop_group_insert:
6872	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6873	err_nexthop_group_info_init:
6874	rhashtable_destroy(ht: &nh_grp->vr_ht);
6875	err_nexthop_group_vr_ht_init:
6876	kfree(objp: nh_grp);
6877	return ERR_PTR(error: err);
6878	}
6879
6880	static void
6881	mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
6882	struct mlxsw_sp_nexthop_group *nh_grp)
6883	{
6884	if (!nh_grp->can_destroy)
6885	return;
6886	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
6887	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6888	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
6889	rhashtable_destroy(ht: &nh_grp->vr_ht);
6890	kfree(objp: nh_grp);
6891	}
6892
6893	static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
6894	struct mlxsw_sp_fib6_entry *fib6_entry)
6895	{
6896	struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6897	struct mlxsw_sp_nexthop_group *nh_grp;
6898
6899	if (rt->nh) {
6900	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
6901	id: rt->nh->id);
6902	if (WARN_ON_ONCE(!nh_grp))
6903	return -EINVAL;
6904	goto out;
6905	}
6906
6907	nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
6908	if (!nh_grp) {
6909	nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
6910	if (IS_ERR(ptr: nh_grp))
6911	return PTR_ERR(ptr: nh_grp);
6912	}
6913
6914	/* The route and the nexthop are described by the same struct, so we
6915	* need to the update the nexthop offload indication for the new route.
6916	*/
6917	__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
6918
6919	out:
6920	list_add_tail(new: &fib6_entry->common.nexthop_group_node,
6921	head: &nh_grp->fib_list);
6922	fib6_entry->common.nh_group = nh_grp;
6923
6924	return 0;
6925	}
6926
6927	static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
6928	struct mlxsw_sp_fib_entry *fib_entry)
6929	{
6930	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
6931
6932	list_del(entry: &fib_entry->nexthop_group_node);
6933	if (!list_empty(head: &nh_grp->fib_list))
6934	return;
6935
6936	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
6937	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
6938	return;
6939	}
6940
6941	mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
6942	}
6943
6944	static int
6945	mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
6946	struct mlxsw_sp_fib6_entry *fib6_entry)
6947	{
6948	struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
6949	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6950	int err;
6951
6952	mlxsw_sp_nexthop_group_vr_unlink(nh_grp: old_nh_grp, fib: fib_node->fib);
6953	fib6_entry->common.nh_group = NULL;
6954	list_del(entry: &fib6_entry->common.nexthop_group_node);
6955
6956	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6957	if (err)
6958	goto err_nexthop6_group_get;
6959
6960	err = mlxsw_sp_nexthop_group_vr_link(nh_grp: fib6_entry->common.nh_group,
6961	fib: fib_node->fib);
6962	if (err)
6963	goto err_nexthop_group_vr_link;
6964
6965	/* In case this entry is offloaded, then the adjacency index
6966	* currently associated with it in the device's table is that
6967	* of the old group. Start using the new one instead.
6968	*/
6969	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry: &fib6_entry->common);
6970	if (err)
6971	goto err_fib_entry_update;
6972
6973	if (list_empty(head: &old_nh_grp->fib_list))
6974	mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp: old_nh_grp);
6975
6976	return 0;
6977
6978	err_fib_entry_update:
6979	mlxsw_sp_nexthop_group_vr_unlink(nh_grp: fib6_entry->common.nh_group,
6980	fib: fib_node->fib);
6981	err_nexthop_group_vr_link:
6982	mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry: &fib6_entry->common);
6983	err_nexthop6_group_get:
6984	list_add_tail(new: &fib6_entry->common.nexthop_group_node,
6985	head: &old_nh_grp->fib_list);
6986	fib6_entry->common.nh_group = old_nh_grp;
6987	mlxsw_sp_nexthop_group_vr_link(nh_grp: old_nh_grp, fib: fib_node->fib);
6988	return err;
6989	}
6990
6991	static int
6992	mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
6993	struct mlxsw_sp_fib6_entry *fib6_entry,
6994	struct fib6_info **rt_arr, unsigned int nrt6)
6995	{
6996	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6997	int err, i;
6998
6999	for (i = 0; i < nrt6; i++) {
7000	mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt: rt_arr[i]);
7001	if (IS_ERR(ptr: mlxsw_sp_rt6)) {
7002	err = PTR_ERR(ptr: mlxsw_sp_rt6);
7003	goto err_rt6_unwind;
7004	}
7005
7006	list_add_tail(new: &mlxsw_sp_rt6->list, head: &fib6_entry->rt6_list);
7007	fib6_entry->nrt6++;
7008	}
7009
7010	err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
7011	if (err)
7012	goto err_rt6_unwind;
7013
7014	return 0;
7015
7016	err_rt6_unwind:
7017	for (; i > 0; i--) {
7018	fib6_entry->nrt6--;
7019	mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
7020	struct mlxsw_sp_rt6, list);
7021	list_del(entry: &mlxsw_sp_rt6->list);
7022	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7023	}
7024	return err;
7025	}
7026
7027	static void
7028	mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
7029	struct mlxsw_sp_fib6_entry *fib6_entry,
7030	struct fib6_info **rt_arr, unsigned int nrt6)
7031	{
7032	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
7033	int i;
7034
7035	for (i = 0; i < nrt6; i++) {
7036	mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
7037	rt: rt_arr[i]);
7038	if (WARN_ON_ONCE(!mlxsw_sp_rt6))
7039	continue;
7040
7041	fib6_entry->nrt6--;
7042	list_del(entry: &mlxsw_sp_rt6->list);
7043	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7044	}
7045
7046	mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
7047	}
7048
7049	static int
7050	mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp,
7051	struct mlxsw_sp_fib_entry *fib_entry,
7052	const struct fib6_info *rt)
7053	{
7054	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
7055	union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr };
7056	u32 tb_id = mlxsw_sp_fix_tb_id(tb_id: rt->fib6_table->tb6_id);
7057	struct mlxsw_sp_router *router = mlxsw_sp->router;
7058	int ifindex = nhgi->nexthops[0].ifindex;
7059	struct mlxsw_sp_ipip_entry *ipip_entry;
7060
7061	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
7062	ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ul_dev_ifindex: ifindex,
7063	ul_proto: MLXSW_SP_L3_PROTO_IPV6,
7064	ul_dip: dip);
7065
7066	if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
7067	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
7068	return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry,
7069	ipip_entry);
7070	}
7071	if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, ul_tb_id: tb_id,
7072	ul_proto: MLXSW_SP_L3_PROTO_IPV6, ul_sip: &dip)) {
7073	u32 tunnel_index;
7074
7075	tunnel_index = router->nve_decap_config.tunnel_index;
7076	fib_entry->decap.tunnel_index = tunnel_index;
7077	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
7078	}
7079
7080	return 0;
7081	}
7082
7083	static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
7084	struct mlxsw_sp_fib_entry *fib_entry,
7085	const struct fib6_info *rt)
7086	{
7087	if (rt->fib6_flags & RTF_LOCAL)
7088	return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry,
7089	rt);
7090	if (rt->fib6_flags & RTF_ANYCAST)
7091	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
7092	else if (rt->fib6_type == RTN_BLACKHOLE)
7093	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
7094	else if (rt->fib6_flags & RTF_REJECT)
7095	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
7096	else if (fib_entry->nh_group->nhgi->gateway)
7097	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
7098	else
7099	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
7100
7101	return 0;
7102	}
7103
7104	static void
7105	mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
7106	{
7107	struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
7108
7109	list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
7110	list) {
7111	fib6_entry->nrt6--;
7112	list_del(entry: &mlxsw_sp_rt6->list);
7113	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7114	}
7115	}
7116
7117	static struct mlxsw_sp_fib6_entry *
7118	mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
7119	struct mlxsw_sp_fib_node *fib_node,
7120	struct fib6_info **rt_arr, unsigned int nrt6)
7121	{
7122	struct mlxsw_sp_fib6_entry *fib6_entry;
7123	struct mlxsw_sp_fib_entry *fib_entry;
7124	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
7125	int err, i;
7126
7127	fib6_entry = kzalloc(size: sizeof(*fib6_entry), GFP_KERNEL);
7128	if (!fib6_entry)
7129	return ERR_PTR(error: -ENOMEM);
7130	fib_entry = &fib6_entry->common;
7131
7132	INIT_LIST_HEAD(list: &fib6_entry->rt6_list);
7133
7134	for (i = 0; i < nrt6; i++) {
7135	mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt: rt_arr[i]);
7136	if (IS_ERR(ptr: mlxsw_sp_rt6)) {
7137	err = PTR_ERR(ptr: mlxsw_sp_rt6);
7138	goto err_rt6_unwind;
7139	}
7140	list_add_tail(new: &mlxsw_sp_rt6->list, head: &fib6_entry->rt6_list);
7141	fib6_entry->nrt6++;
7142	}
7143
7144	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
7145	if (err)
7146	goto err_rt6_unwind;
7147
7148	err = mlxsw_sp_nexthop_group_vr_link(nh_grp: fib_entry->nh_group,
7149	fib: fib_node->fib);
7150	if (err)
7151	goto err_nexthop_group_vr_link;
7152
7153	err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt: rt_arr[0]);
7154	if (err)
7155	goto err_fib6_entry_type_set;
7156
7157	fib_entry->fib_node = fib_node;
7158
7159	return fib6_entry;
7160
7161	err_fib6_entry_type_set:
7162	mlxsw_sp_nexthop_group_vr_unlink(nh_grp: fib_entry->nh_group, fib: fib_node->fib);
7163	err_nexthop_group_vr_link:
7164	mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry);
7165	err_rt6_unwind:
7166	for (; i > 0; i--) {
7167	fib6_entry->nrt6--;
7168	mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
7169	struct mlxsw_sp_rt6, list);
7170	list_del(entry: &mlxsw_sp_rt6->list);
7171	mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7172	}
7173	kfree(objp: fib6_entry);
7174	return ERR_PTR(error: err);
7175	}
7176
7177	static void
7178	mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
7179	struct mlxsw_sp_fib6_entry *fib6_entry)
7180	{
7181	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, fib_entry: &fib6_entry->common);
7182	}
7183
7184	static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
7185	struct mlxsw_sp_fib6_entry *fib6_entry)
7186	{
7187	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7188
7189	mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry);
7190	mlxsw_sp_nexthop_group_vr_unlink(nh_grp: fib6_entry->common.nh_group,
7191	fib: fib_node->fib);
7192	mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry: &fib6_entry->common);
7193	mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
7194	WARN_ON(fib6_entry->nrt6);
7195	kfree(objp: fib6_entry);
7196	}
7197
7198	static struct mlxsw_sp_fib6_entry *
7199	mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
7200	const struct fib6_info *rt)
7201	{
7202	struct mlxsw_sp_fib6_entry *fib6_entry;
7203	struct mlxsw_sp_fib_node *fib_node;
7204	struct mlxsw_sp_fib *fib;
7205	struct fib6_info *cmp_rt;
7206	struct mlxsw_sp_vr *vr;
7207
7208	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id: rt->fib6_table->tb6_id);
7209	if (!vr)
7210	return NULL;
7211	fib = mlxsw_sp_vr_fib(vr, proto: MLXSW_SP_L3_PROTO_IPV6);
7212
7213	fib_node = mlxsw_sp_fib_node_lookup(fib, addr: &rt->fib6_dst.addr,
7214	addr_len: sizeof(rt->fib6_dst.addr),
7215	prefix_len: rt->fib6_dst.plen);
7216	if (!fib_node)
7217	return NULL;
7218
7219	fib6_entry = container_of(fib_node->fib_entry,
7220	struct mlxsw_sp_fib6_entry, common);
7221	cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7222	if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
7223	rt->fib6_metric == cmp_rt->fib6_metric &&
7224	mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
7225	return fib6_entry;
7226
7227	return NULL;
7228	}
7229
7230	static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
7231	{
7232	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7233	struct mlxsw_sp_fib6_entry *fib6_replaced;
7234	struct fib6_info *rt, *rt_replaced;
7235
7236	if (!fib_node->fib_entry)
7237	return true;
7238
7239	fib6_replaced = container_of(fib_node->fib_entry,
7240	struct mlxsw_sp_fib6_entry,
7241	common);
7242	rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7243	rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_entry: fib6_replaced);
7244	if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
7245	rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
7246	return false;
7247
7248	return true;
7249	}
7250
7251	static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
7252	struct fib6_info **rt_arr,
7253	unsigned int nrt6)
7254	{
7255	struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
7256	struct mlxsw_sp_fib_entry *replaced;
7257	struct mlxsw_sp_fib_node *fib_node;
7258	struct fib6_info *rt = rt_arr[0];
7259	int err;
7260
7261	if (rt->fib6_src.plen)
7262	return -EINVAL;
7263
7264	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7265	return 0;
7266
7267	if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, id: rt->nh->id))
7268	return 0;
7269
7270	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, tb_id: rt->fib6_table->tb6_id,
7271	addr: &rt->fib6_dst.addr,
7272	addr_len: sizeof(rt->fib6_dst.addr),
7273	prefix_len: rt->fib6_dst.plen,
7274	proto: MLXSW_SP_L3_PROTO_IPV6);
7275	if (IS_ERR(ptr: fib_node))
7276	return PTR_ERR(ptr: fib_node);
7277
7278	fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
7279	nrt6);
7280	if (IS_ERR(ptr: fib6_entry)) {
7281	err = PTR_ERR(ptr: fib6_entry);
7282	goto err_fib6_entry_create;
7283	}
7284
7285	if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
7286	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7287	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7288	return 0;
7289	}
7290
7291	replaced = fib_node->fib_entry;
7292	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, fib_entry: &fib6_entry->common);
7293	if (err)
7294	goto err_fib_node_entry_link;
7295
7296	/* Nothing to replace */
7297	if (!replaced)
7298	return 0;
7299
7300	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry: replaced);
7301	fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
7302	common);
7303	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry: fib6_replaced);
7304
7305	return 0;
7306
7307	err_fib_node_entry_link:
7308	fib_node->fib_entry = replaced;
7309	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7310	err_fib6_entry_create:
7311	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7312	return err;
7313	}
7314
7315	static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
7316	struct fib6_info **rt_arr,
7317	unsigned int nrt6)
7318	{
7319	struct mlxsw_sp_fib6_entry *fib6_entry;
7320	struct mlxsw_sp_fib_node *fib_node;
7321	struct fib6_info *rt = rt_arr[0];
7322	int err;
7323
7324	if (rt->fib6_src.plen)
7325	return -EINVAL;
7326
7327	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7328	return 0;
7329
7330	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, tb_id: rt->fib6_table->tb6_id,
7331	addr: &rt->fib6_dst.addr,
7332	addr_len: sizeof(rt->fib6_dst.addr),
7333	prefix_len: rt->fib6_dst.plen,
7334	proto: MLXSW_SP_L3_PROTO_IPV6);
7335	if (IS_ERR(ptr: fib_node))
7336	return PTR_ERR(ptr: fib_node);
7337
7338	if (WARN_ON_ONCE(!fib_node->fib_entry)) {
7339	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7340	return -EINVAL;
7341	}
7342
7343	fib6_entry = container_of(fib_node->fib_entry,
7344	struct mlxsw_sp_fib6_entry, common);
7345	err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr,
7346	nrt6);
7347	if (err)
7348	goto err_fib6_entry_nexthop_add;
7349
7350	return 0;
7351
7352	err_fib6_entry_nexthop_add:
7353	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7354	return err;
7355	}
7356
7357	static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
7358	struct fib6_info **rt_arr,
7359	unsigned int nrt6)
7360	{
7361	struct mlxsw_sp_fib6_entry *fib6_entry;
7362	struct mlxsw_sp_fib_node *fib_node;
7363	struct fib6_info *rt = rt_arr[0];
7364
7365	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7366	return;
7367
7368	/* Multipath routes are first added to the FIB trie and only then
7369	* notified. If we vetoed the addition, we will get a delete
7370	* notification for a route we do not have. Therefore, do not warn if
7371	* route was not found.
7372	*/
7373	fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
7374	if (!fib6_entry)
7375	return;
7376
7377	/* If not all the nexthops are deleted, then only reduce the nexthop
7378	* group.
7379	*/
7380	if (nrt6 != fib6_entry->nrt6) {
7381	mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr,
7382	nrt6);
7383	return;
7384	}
7385
7386	fib_node = fib6_entry->common.fib_node;
7387
7388	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_entry: &fib6_entry->common);
7389	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7390	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7391	}
7392
7393	static struct mlxsw_sp_mr_table *
7394	mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
7395	{
7396	if (family == RTNL_FAMILY_IPMR)
7397	return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
7398	else
7399	return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
7400	}
7401
7402	static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
7403	struct mfc_entry_notifier_info *men_info,
7404	bool replace)
7405	{
7406	struct mlxsw_sp_mr_table *mrt;
7407	struct mlxsw_sp_vr *vr;
7408
7409	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id: men_info->tb_id, NULL);
7410	if (IS_ERR(ptr: vr))
7411	return PTR_ERR(ptr: vr);
7412
7413	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, family: men_info->info.family);
7414	return mlxsw_sp_mr_route_add(mr_table: mrt, mfc: men_info->mfc, replace);
7415	}
7416
7417	static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
7418	struct mfc_entry_notifier_info *men_info)
7419	{
7420	struct mlxsw_sp_mr_table *mrt;
7421	struct mlxsw_sp_vr *vr;
7422
7423	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id: men_info->tb_id);
7424	if (WARN_ON(!vr))
7425	return;
7426
7427	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, family: men_info->info.family);
7428	mlxsw_sp_mr_route_del(mr_table: mrt, mfc: men_info->mfc);
7429	mlxsw_sp_vr_put(mlxsw_sp, vr);
7430	}
7431
7432	static int
7433	mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
7434	struct vif_entry_notifier_info *ven_info)
7435	{
7436	struct mlxsw_sp_mr_table *mrt;
7437	struct mlxsw_sp_rif *rif;
7438	struct mlxsw_sp_vr *vr;
7439
7440	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id: ven_info->tb_id, NULL);
7441	if (IS_ERR(ptr: vr))
7442	return PTR_ERR(ptr: vr);
7443
7444	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, family: ven_info->info.family);
7445	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: ven_info->dev);
7446	return mlxsw_sp_mr_vif_add(mr_table: mrt, dev: ven_info->dev,
7447	vif_index: ven_info->vif_index,
7448	vif_flags: ven_info->vif_flags, rif);
7449	}
7450
7451	static void
7452	mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
7453	struct vif_entry_notifier_info *ven_info)
7454	{
7455	struct mlxsw_sp_mr_table *mrt;
7456	struct mlxsw_sp_vr *vr;
7457
7458	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id: ven_info->tb_id);
7459	if (WARN_ON(!vr))
7460	return;
7461
7462	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, family: ven_info->info.family);
7463	mlxsw_sp_mr_vif_del(mr_table: mrt, vif_index: ven_info->vif_index);
7464	mlxsw_sp_vr_put(mlxsw_sp, vr);
7465	}
7466
7467	static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
7468	struct mlxsw_sp_fib_node *fib_node)
7469	{
7470	struct mlxsw_sp_fib4_entry *fib4_entry;
7471
7472	fib4_entry = container_of(fib_node->fib_entry,
7473	struct mlxsw_sp_fib4_entry, common);
7474	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_entry: fib_node->fib_entry);
7475	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
7476	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7477	}
7478
7479	static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
7480	struct mlxsw_sp_fib_node *fib_node)
7481	{
7482	struct mlxsw_sp_fib6_entry *fib6_entry;
7483
7484	fib6_entry = container_of(fib_node->fib_entry,
7485	struct mlxsw_sp_fib6_entry, common);
7486	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_entry: fib_node->fib_entry);
7487	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7488	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7489	}
7490
7491	static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
7492	struct mlxsw_sp_fib_node *fib_node)
7493	{
7494	switch (fib_node->fib->proto) {
7495	case MLXSW_SP_L3_PROTO_IPV4:
7496	mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
7497	break;
7498	case MLXSW_SP_L3_PROTO_IPV6:
7499	mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
7500	break;
7501	}
7502	}
7503
7504	static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
7505	struct mlxsw_sp_vr *vr,
7506	enum mlxsw_sp_l3proto proto)
7507	{
7508	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
7509	struct mlxsw_sp_fib_node *fib_node, *tmp;
7510
7511	list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
7512	bool do_break = &tmp->list == &fib->node_list;
7513
7514	mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
7515	if (do_break)
7516	break;
7517	}
7518	}
7519
7520	static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
7521	{
7522	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
7523	int i, j;
7524
7525	for (i = 0; i < max_vrs; i++) {
7526	struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
7527
7528	if (!mlxsw_sp_vr_is_used(vr))
7529	continue;
7530
7531	for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
7532	mlxsw_sp_mr_table_flush(mr_table: vr->mr_table[j]);
7533	mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, proto: MLXSW_SP_L3_PROTO_IPV4);
7534
7535	/* If virtual router was only used for IPv4, then it's no
7536	* longer used.
7537	*/
7538	if (!mlxsw_sp_vr_is_used(vr))
7539	continue;
7540	mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, proto: MLXSW_SP_L3_PROTO_IPV6);
7541	}
7542	}
7543
7544	struct mlxsw_sp_fib6_event_work {
7545	struct fib6_info **rt_arr;
7546	unsigned int nrt6;
7547	};
7548
7549	struct mlxsw_sp_fib_event_work {
7550	struct work_struct work;
7551	netdevice_tracker dev_tracker;
7552	union {
7553	struct mlxsw_sp_fib6_event_work fib6_work;
7554	struct fib_entry_notifier_info fen_info;
7555	struct fib_rule_notifier_info fr_info;
7556	struct fib_nh_notifier_info fnh_info;
7557	struct mfc_entry_notifier_info men_info;
7558	struct vif_entry_notifier_info ven_info;
7559	};
7560	struct mlxsw_sp *mlxsw_sp;
7561	unsigned long event;
7562	};
7563
7564	static int
7565	mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work,
7566	struct fib6_entry_notifier_info *fen6_info)
7567	{
7568	struct fib6_info *rt = fen6_info->rt;
7569	struct fib6_info **rt_arr;
7570	struct fib6_info *iter;
7571	unsigned int nrt6;
7572	int i = 0;
7573
7574	nrt6 = fen6_info->nsiblings + 1;
7575
7576	rt_arr = kcalloc(n: nrt6, size: sizeof(struct fib6_info *), GFP_ATOMIC);
7577	if (!rt_arr)
7578	return -ENOMEM;
7579
7580	fib6_work->rt_arr = rt_arr;
7581	fib6_work->nrt6 = nrt6;
7582
7583	rt_arr[0] = rt;
7584	fib6_info_hold(f6i: rt);
7585
7586	if (!fen6_info->nsiblings)
7587	return 0;
7588
7589	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
7590	if (i == fen6_info->nsiblings)
7591	break;
7592
7593	rt_arr[i + 1] = iter;
7594	fib6_info_hold(f6i: iter);
7595	i++;
7596	}
7597	WARN_ON_ONCE(i != fen6_info->nsiblings);
7598
7599	return 0;
7600	}
7601
7602	static void
7603	mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work)
7604	{
7605	int i;
7606
7607	for (i = 0; i < fib6_work->nrt6; i++)
7608	mlxsw_sp_rt6_release(rt: fib6_work->rt_arr[i]);
7609	kfree(objp: fib6_work->rt_arr);
7610	}
7611
7612	static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
7613	{
7614	struct mlxsw_sp_fib_event_work *fib_work =
7615	container_of(work, struct mlxsw_sp_fib_event_work, work);
7616	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7617	int err;
7618
7619	mutex_lock(&mlxsw_sp->router->lock);
7620	mlxsw_sp_span_respin(mlxsw_sp);
7621
7622	switch (fib_work->event) {
7623	case FIB_EVENT_ENTRY_REPLACE:
7624	err = mlxsw_sp_router_fib4_replace(mlxsw_sp,
7625	fen_info: &fib_work->fen_info);
7626	if (err) {
7627	dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7628	mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp,
7629	fen_info: &fib_work->fen_info);
7630	}
7631	fib_info_put(fi: fib_work->fen_info.fi);
7632	break;
7633	case FIB_EVENT_ENTRY_DEL:
7634	mlxsw_sp_router_fib4_del(mlxsw_sp, fen_info: &fib_work->fen_info);
7635	fib_info_put(fi: fib_work->fen_info.fi);
7636	break;
7637	case FIB_EVENT_NH_ADD:
7638	case FIB_EVENT_NH_DEL:
7639	mlxsw_sp_nexthop4_event(mlxsw_sp, event: fib_work->event,
7640	fib_nh: fib_work->fnh_info.fib_nh);
7641	fib_info_put(fi: fib_work->fnh_info.fib_nh->nh_parent);
7642	break;
7643	}
7644	mutex_unlock(lock: &mlxsw_sp->router->lock);
7645	kfree(objp: fib_work);
7646	}
7647
7648	static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
7649	{
7650	struct mlxsw_sp_fib_event_work *fib_work =
7651	container_of(work, struct mlxsw_sp_fib_event_work, work);
7652	struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work;
7653	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7654	int err;
7655
7656	mutex_lock(&mlxsw_sp->router->lock);
7657	mlxsw_sp_span_respin(mlxsw_sp);
7658
7659	switch (fib_work->event) {
7660	case FIB_EVENT_ENTRY_REPLACE:
7661	err = mlxsw_sp_router_fib6_replace(mlxsw_sp,
7662	rt_arr: fib6_work->rt_arr,
7663	nrt6: fib6_work->nrt6);
7664	if (err) {
7665	dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7666	mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7667	rt_arr: fib6_work->rt_arr,
7668	nrt6: fib6_work->nrt6);
7669	}
7670	mlxsw_sp_router_fib6_work_fini(fib6_work);
7671	break;
7672	case FIB_EVENT_ENTRY_APPEND:
7673	err = mlxsw_sp_router_fib6_append(mlxsw_sp,
7674	rt_arr: fib6_work->rt_arr,
7675	nrt6: fib6_work->nrt6);
7676	if (err) {
7677	dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n");
7678	mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7679	rt_arr: fib6_work->rt_arr,
7680	nrt6: fib6_work->nrt6);
7681	}
7682	mlxsw_sp_router_fib6_work_fini(fib6_work);
7683	break;
7684	case FIB_EVENT_ENTRY_DEL:
7685	mlxsw_sp_router_fib6_del(mlxsw_sp,
7686	rt_arr: fib6_work->rt_arr,
7687	nrt6: fib6_work->nrt6);
7688	mlxsw_sp_router_fib6_work_fini(fib6_work);
7689	break;
7690	}
7691	mutex_unlock(lock: &mlxsw_sp->router->lock);
7692	kfree(objp: fib_work);
7693	}
7694
7695	static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
7696	{
7697	struct mlxsw_sp_fib_event_work *fib_work =
7698	container_of(work, struct mlxsw_sp_fib_event_work, work);
7699	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7700	bool replace;
7701	int err;
7702
7703	rtnl_lock();
7704	mutex_lock(&mlxsw_sp->router->lock);
7705	switch (fib_work->event) {
7706	case FIB_EVENT_ENTRY_REPLACE:
7707	case FIB_EVENT_ENTRY_ADD:
7708	replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
7709
7710	err = mlxsw_sp_router_fibmr_add(mlxsw_sp, men_info: &fib_work->men_info,
7711	replace);
7712	if (err)
7713	dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n");
7714	mr_cache_put(c: fib_work->men_info.mfc);
7715	break;
7716	case FIB_EVENT_ENTRY_DEL:
7717	mlxsw_sp_router_fibmr_del(mlxsw_sp, men_info: &fib_work->men_info);
7718	mr_cache_put(c: fib_work->men_info.mfc);
7719	break;
7720	case FIB_EVENT_VIF_ADD:
7721	err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
7722	ven_info: &fib_work->ven_info);
7723	if (err)
7724	dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n");
7725	netdev_put(dev: fib_work->ven_info.dev, tracker: &fib_work->dev_tracker);
7726	break;
7727	case FIB_EVENT_VIF_DEL:
7728	mlxsw_sp_router_fibmr_vif_del(mlxsw_sp,
7729	ven_info: &fib_work->ven_info);
7730	netdev_put(dev: fib_work->ven_info.dev, tracker: &fib_work->dev_tracker);
7731	break;
7732	}
7733	mutex_unlock(lock: &mlxsw_sp->router->lock);
7734	rtnl_unlock();
7735	kfree(objp: fib_work);
7736	}
7737
7738	static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
7739	struct fib_notifier_info *info)
7740	{
7741	struct fib_entry_notifier_info *fen_info;
7742	struct fib_nh_notifier_info *fnh_info;
7743
7744	switch (fib_work->event) {
7745	case FIB_EVENT_ENTRY_REPLACE:
7746	case FIB_EVENT_ENTRY_DEL:
7747	fen_info = container_of(info, struct fib_entry_notifier_info,
7748	info);
7749	fib_work->fen_info = *fen_info;
7750	/* Take reference on fib_info to prevent it from being
7751	* freed while work is queued. Release it afterwards.
7752	*/
7753	fib_info_hold(fi: fib_work->fen_info.fi);
7754	break;
7755	case FIB_EVENT_NH_ADD:
7756	case FIB_EVENT_NH_DEL:
7757	fnh_info = container_of(info, struct fib_nh_notifier_info,
7758	info);
7759	fib_work->fnh_info = *fnh_info;
7760	fib_info_hold(fi: fib_work->fnh_info.fib_nh->nh_parent);
7761	break;
7762	}
7763	}
7764
7765	static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
7766	struct fib_notifier_info *info)
7767	{
7768	struct fib6_entry_notifier_info *fen6_info;
7769	int err;
7770
7771	switch (fib_work->event) {
7772	case FIB_EVENT_ENTRY_REPLACE:
7773	case FIB_EVENT_ENTRY_APPEND:
7774	case FIB_EVENT_ENTRY_DEL:
7775	fen6_info = container_of(info, struct fib6_entry_notifier_info,
7776	info);
7777	err = mlxsw_sp_router_fib6_work_init(fib6_work: &fib_work->fib6_work,
7778	fen6_info);
7779	if (err)
7780	return err;
7781	break;
7782	}
7783
7784	return 0;
7785	}
7786
7787	static void
7788	mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
7789	struct fib_notifier_info *info)
7790	{
7791	switch (fib_work->event) {
7792	case FIB_EVENT_ENTRY_REPLACE:
7793	case FIB_EVENT_ENTRY_ADD:
7794	case FIB_EVENT_ENTRY_DEL:
7795	memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info));
7796	mr_cache_hold(c: fib_work->men_info.mfc);
7797	break;
7798	case FIB_EVENT_VIF_ADD:
7799	case FIB_EVENT_VIF_DEL:
7800	memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
7801	netdev_hold(dev: fib_work->ven_info.dev, tracker: &fib_work->dev_tracker,
7802	GFP_ATOMIC);
7803	break;
7804	}
7805	}
7806
7807	static int mlxsw_sp_router_fib_rule_event(unsigned long event,
7808	struct fib_notifier_info *info,
7809	struct mlxsw_sp *mlxsw_sp)
7810	{
7811	struct netlink_ext_ack *extack = info->extack;
7812	struct fib_rule_notifier_info *fr_info;
7813	struct fib_rule *rule;
7814	int err = 0;
7815
7816	/* nothing to do at the moment */
7817	if (event == FIB_EVENT_RULE_DEL)
7818	return 0;
7819
7820	fr_info = container_of(info, struct fib_rule_notifier_info, info);
7821	rule = fr_info->rule;
7822
7823	/* Rule only affects locally generated traffic */
7824	if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
7825	return 0;
7826
7827	switch (info->family) {
7828	case AF_INET:
7829	if (!fib4_rule_default(rule) && !rule->l3mdev)
7830	err = -EOPNOTSUPP;
7831	break;
7832	case AF_INET6:
7833	if (!fib6_rule_default(rule) && !rule->l3mdev)
7834	err = -EOPNOTSUPP;
7835	break;
7836	case RTNL_FAMILY_IPMR:
7837	if (!ipmr_rule_default(rule) && !rule->l3mdev)
7838	err = -EOPNOTSUPP;
7839	break;
7840	case RTNL_FAMILY_IP6MR:
7841	if (!ip6mr_rule_default(rule) && !rule->l3mdev)
7842	err = -EOPNOTSUPP;
7843	break;
7844	}
7845
7846	if (err < 0)
7847	NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
7848
7849	return err;
7850	}
7851
7852	/* Called with rcu_read_lock() */
7853	static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
7854	unsigned long event, void *ptr)
7855	{
7856	struct mlxsw_sp_fib_event_work *fib_work;
7857	struct fib_notifier_info *info = ptr;
7858	struct mlxsw_sp_router *router;
7859	int err;
7860
7861	if ((info->family != AF_INET && info->family != AF_INET6 &&
7862	info->family != RTNL_FAMILY_IPMR &&
7863	info->family != RTNL_FAMILY_IP6MR))
7864	return NOTIFY_DONE;
7865
7866	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7867
7868	switch (event) {
7869	case FIB_EVENT_RULE_ADD:
7870	case FIB_EVENT_RULE_DEL:
7871	err = mlxsw_sp_router_fib_rule_event(event, info,
7872	mlxsw_sp: router->mlxsw_sp);
7873	return notifier_from_errno(err);
7874	case FIB_EVENT_ENTRY_ADD:
7875	case FIB_EVENT_ENTRY_REPLACE:
7876	case FIB_EVENT_ENTRY_APPEND:
7877	if (info->family == AF_INET) {
7878	struct fib_entry_notifier_info *fen_info = ptr;
7879
7880	if (fen_info->fi->fib_nh_is_v6) {
7881	NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
7882	return notifier_from_errno(err: -EINVAL);
7883	}
7884	}
7885	break;
7886	}
7887
7888	fib_work = kzalloc(size: sizeof(*fib_work), GFP_ATOMIC);
7889	if (!fib_work)
7890	return NOTIFY_BAD;
7891
7892	fib_work->mlxsw_sp = router->mlxsw_sp;
7893	fib_work->event = event;
7894
7895	switch (info->family) {
7896	case AF_INET:
7897	INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
7898	mlxsw_sp_router_fib4_event(fib_work, info);
7899	break;
7900	case AF_INET6:
7901	INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
7902	err = mlxsw_sp_router_fib6_event(fib_work, info);
7903	if (err)
7904	goto err_fib_event;
7905	break;
7906	case RTNL_FAMILY_IP6MR:
7907	case RTNL_FAMILY_IPMR:
7908	INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work);
7909	mlxsw_sp_router_fibmr_event(fib_work, info);
7910	break;
7911	}
7912
7913	mlxsw_core_schedule_work(work: &fib_work->work);
7914
7915	return NOTIFY_DONE;
7916
7917	err_fib_event:
7918	kfree(objp: fib_work);
7919	return NOTIFY_BAD;
7920	}
7921
7922	static struct mlxsw_sp_rif *
7923	mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
7924	const struct net_device *dev)
7925	{
7926	int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7927	int i;
7928
7929	for (i = 0; i < max_rifs; i++)
7930	if (mlxsw_sp->router->rifs[i] &&
7931	mlxsw_sp_rif_dev_is(rif: mlxsw_sp->router->rifs[i], dev))
7932	return mlxsw_sp->router->rifs[i];
7933
7934	return NULL;
7935	}
7936
7937	bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
7938	const struct net_device *dev)
7939	{
7940	struct mlxsw_sp_rif *rif;
7941
7942	mutex_lock(&mlxsw_sp->router->lock);
7943	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7944	mutex_unlock(lock: &mlxsw_sp->router->lock);
7945
7946	return rif;
7947	}
7948
7949	u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
7950	{
7951	struct mlxsw_sp_rif *rif;
7952	u16 vid = 0;
7953
7954	mutex_lock(&mlxsw_sp->router->lock);
7955	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7956	if (!rif)
7957	goto out;
7958
7959	/* We only return the VID for VLAN RIFs. Otherwise we return an
7960	* invalid value (0).
7961	*/
7962	if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
7963	goto out;
7964
7965	vid = mlxsw_sp_fid_8021q_vid(fid: rif->fid);
7966
7967	out:
7968	mutex_unlock(lock: &mlxsw_sp->router->lock);
7969	return vid;
7970	}
7971
7972	static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
7973	{
7974	char ritr_pl[MLXSW_REG_RITR_LEN];
7975	int err;
7976
7977	mlxsw_reg_ritr_rif_pack(payload: ritr_pl, rif);
7978	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
7979	if (err)
7980	return err;
7981
7982	mlxsw_reg_ritr_enable_set(buf: ritr_pl, val: false);
7983	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
7984	}
7985
7986	static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
7987	struct mlxsw_sp_rif *rif)
7988	{
7989	int err;
7990
7991	err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif);
7992	if (err)
7993	return err;
7994
7995	err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif);
7996	if (err)
7997	goto err_nexthop;
7998
7999	return 0;
8000
8001	err_nexthop:
8002	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
8003	return err;
8004	}
8005
8006	static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
8007	struct mlxsw_sp_rif *rif)
8008	{
8009	/* Signal to nexthop cleanup that the RIF is going away. */
8010	rif->crif->rif = NULL;
8011
8012	mlxsw_sp_router_rif_disable(mlxsw_sp, rif: rif->rif_index);
8013	mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
8014	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
8015	}
8016
8017	static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev)
8018	{
8019	struct inet6_dev *inet6_dev;
8020	struct in_device *idev;
8021
8022	idev = __in_dev_get_rcu(dev);
8023	if (idev && idev->ifa_list)
8024	return false;
8025
8026	inet6_dev = __in6_dev_get(dev);
8027	if (inet6_dev && !list_empty(head: &inet6_dev->addr_list))
8028	return false;
8029
8030	return true;
8031	}
8032
8033	static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev)
8034	{
8035	bool addr_list_empty;
8036
8037	rcu_read_lock();
8038	addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev);
8039	rcu_read_unlock();
8040
8041	return addr_list_empty;
8042	}
8043
8044	static bool
8045	mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
8046	unsigned long event)
8047	{
8048	bool addr_list_empty;
8049
8050	switch (event) {
8051	case NETDEV_UP:
8052	return rif == NULL;
8053	case NETDEV_DOWN:
8054	addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev);
8055
8056	/* macvlans do not have a RIF, but rather piggy back on the
8057	* RIF of their lower device.
8058	*/
8059	if (netif_is_macvlan(dev) && addr_list_empty)
8060	return true;
8061
8062	if (rif && addr_list_empty &&
8063	!netif_is_l3_slave(dev: mlxsw_sp_rif_dev(rif)))
8064	return true;
8065	/* It is possible we already removed the RIF ourselves
8066	* if it was assigned to a netdev that is now a bridge
8067	* or LAG slave.
8068	*/
8069	return false;
8070	}
8071
8072	return false;
8073	}
8074
8075	static enum mlxsw_sp_rif_type
8076	mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
8077	const struct net_device *dev)
8078	{
8079	enum mlxsw_sp_fid_type type;
8080
8081	if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
8082	return MLXSW_SP_RIF_TYPE_IPIP_LB;
8083
8084	/* Otherwise RIF type is derived from the type of the underlying FID. */
8085	if (is_vlan_dev(dev) && netif_is_bridge_master(dev: vlan_dev_real_dev(dev)))
8086	type = MLXSW_SP_FID_TYPE_8021Q;
8087	else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
8088	type = MLXSW_SP_FID_TYPE_8021Q;
8089	else if (netif_is_bridge_master(dev))
8090	type = MLXSW_SP_FID_TYPE_8021D;
8091	else
8092	type = MLXSW_SP_FID_TYPE_RFID;
8093
8094	return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
8095	}
8096
8097	static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index,
8098	u8 rif_entries)
8099	{
8100	*p_rif_index = gen_pool_alloc(pool: mlxsw_sp->router->rifs_table,
8101	size: rif_entries);
8102	if (*p_rif_index == 0)
8103	return -ENOBUFS;
8104	*p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET;
8105
8106	/* RIF indexes must be aligned to the allocation size. */
8107	WARN_ON_ONCE(*p_rif_index % rif_entries);
8108
8109	return 0;
8110	}
8111
8112	static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8113	u8 rif_entries)
8114	{
8115	gen_pool_free(pool: mlxsw_sp->router->rifs_table,
8116	MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, size: rif_entries);
8117	}
8118
8119	static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
8120	u16 vr_id,
8121	struct mlxsw_sp_crif *crif)
8122	{
8123	struct net_device *l3_dev = crif ? crif->key.dev : NULL;
8124	struct mlxsw_sp_rif *rif;
8125
8126	rif = kzalloc(size: rif_size, GFP_KERNEL);
8127	if (!rif)
8128	return NULL;
8129
8130	INIT_LIST_HEAD(list: &rif->neigh_list);
8131	if (l3_dev) {
8132	ether_addr_copy(dst: rif->addr, src: l3_dev->dev_addr);
8133	rif->mtu = l3_dev->mtu;
8134	}
8135	rif->vr_id = vr_id;
8136	rif->rif_index = rif_index;
8137	if (crif) {
8138	rif->crif = crif;
8139	crif->rif = rif;
8140	}
8141
8142	return rif;
8143	}
8144
8145	static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif)
8146	{
8147	WARN_ON(!list_empty(&rif->neigh_list));
8148
8149	if (rif->crif)
8150	rif->crif->rif = NULL;
8151	kfree(objp: rif);
8152	}
8153
8154	struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
8155	u16 rif_index)
8156	{
8157	return mlxsw_sp->router->rifs[rif_index];
8158	}
8159
8160	u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
8161	{
8162	return rif->rif_index;
8163	}
8164
8165	u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8166	{
8167	return lb_rif->common.rif_index;
8168	}
8169
8170	u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8171	{
8172	struct net_device *dev = mlxsw_sp_rif_dev(rif: &lb_rif->common);
8173	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev: dev);
8174	struct mlxsw_sp_vr *ul_vr;
8175
8176	ul_vr = mlxsw_sp_vr_get(mlxsw_sp: lb_rif->common.mlxsw_sp, tb_id: ul_tb_id, NULL);
8177	if (WARN_ON(IS_ERR(ul_vr)))
8178	return 0;
8179
8180	return ul_vr->id;
8181	}
8182
8183	u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8184	{
8185	return lb_rif->ul_rif_id;
8186	}
8187
8188	static bool
8189	mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif)
8190	{
8191	return mlxsw_sp_rif_counter_valid_get(rif,
8192	dir: MLXSW_SP_RIF_COUNTER_EGRESS) &&
8193	mlxsw_sp_rif_counter_valid_get(rif,
8194	dir: MLXSW_SP_RIF_COUNTER_INGRESS);
8195	}
8196
8197	static int
8198	mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif)
8199	{
8200	int err;
8201
8202	err = mlxsw_sp_rif_counter_alloc(rif, dir: MLXSW_SP_RIF_COUNTER_INGRESS);
8203	if (err)
8204	return err;
8205
8206	/* Clear stale data. */
8207	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8208	dir: MLXSW_SP_RIF_COUNTER_INGRESS,
8209	NULL);
8210	if (err)
8211	goto err_clear_ingress;
8212
8213	err = mlxsw_sp_rif_counter_alloc(rif, dir: MLXSW_SP_RIF_COUNTER_EGRESS);
8214	if (err)
8215	goto err_alloc_egress;
8216
8217	/* Clear stale data. */
8218	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8219	dir: MLXSW_SP_RIF_COUNTER_EGRESS,
8220	NULL);
8221	if (err)
8222	goto err_clear_egress;
8223
8224	return 0;
8225
8226	err_clear_egress:
8227	mlxsw_sp_rif_counter_free(rif, dir: MLXSW_SP_RIF_COUNTER_EGRESS);
8228	err_alloc_egress:
8229	err_clear_ingress:
8230	mlxsw_sp_rif_counter_free(rif, dir: MLXSW_SP_RIF_COUNTER_INGRESS);
8231	return err;
8232	}
8233
8234	static void
8235	mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif)
8236	{
8237	mlxsw_sp_rif_counter_free(rif, dir: MLXSW_SP_RIF_COUNTER_EGRESS);
8238	mlxsw_sp_rif_counter_free(rif, dir: MLXSW_SP_RIF_COUNTER_INGRESS);
8239	}
8240
8241	static void
8242	mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif,
8243	struct netdev_notifier_offload_xstats_info *info)
8244	{
8245	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8246	return;
8247	netdev_offload_xstats_report_used(ru: info->report_used);
8248	}
8249
8250	static int
8251	mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif,
8252	struct rtnl_hw_stats64 *p_stats)
8253	{
8254	struct mlxsw_sp_rif_counter_set_basic ingress;
8255	struct mlxsw_sp_rif_counter_set_basic egress;
8256	int err;
8257
8258	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8259	dir: MLXSW_SP_RIF_COUNTER_INGRESS,
8260	set: &ingress);
8261	if (err)
8262	return err;
8263
8264	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8265	dir: MLXSW_SP_RIF_COUNTER_EGRESS,
8266	set: &egress);
8267	if (err)
8268	return err;
8269
8270	#define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \
8271	((SET.good_unicast_ ## SFX) + \
8272	(SET.good_multicast_ ## SFX) + \
8273	(SET.good_broadcast_ ## SFX))
8274
8275	p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets);
8276	p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets);
8277	p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes);
8278	p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes);
8279	p_stats->rx_errors = ingress.error_packets;
8280	p_stats->tx_errors = egress.error_packets;
8281	p_stats->rx_dropped = ingress.discard_packets;
8282	p_stats->tx_dropped = egress.discard_packets;
8283	p_stats->multicast = ingress.good_multicast_packets +
8284	ingress.good_broadcast_packets;
8285
8286	#undef MLXSW_SP_ROUTER_ALL_GOOD
8287
8288	return 0;
8289	}
8290
8291	static int
8292	mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif,
8293	struct netdev_notifier_offload_xstats_info *info)
8294	{
8295	struct rtnl_hw_stats64 stats = {};
8296	int err;
8297
8298	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8299	return 0;
8300
8301	err = mlxsw_sp_router_port_l3_stats_fetch(rif, p_stats: &stats);
8302	if (err)
8303	return err;
8304
8305	netdev_offload_xstats_report_delta(rd: info->report_delta, stats: &stats);
8306	return 0;
8307	}
8308
8309	struct mlxsw_sp_router_hwstats_notify_work {
8310	struct work_struct work;
8311	struct net_device *dev;
8312	netdevice_tracker dev_tracker;
8313	};
8314
8315	static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work)
8316	{
8317	struct mlxsw_sp_router_hwstats_notify_work *hws_work =
8318	container_of(work, struct mlxsw_sp_router_hwstats_notify_work,
8319	work);
8320
8321	rtnl_lock();
8322	rtnl_offload_xstats_notify(dev: hws_work->dev);
8323	rtnl_unlock();
8324	netdev_put(dev: hws_work->dev, tracker: &hws_work->dev_tracker);
8325	kfree(objp: hws_work);
8326	}
8327
8328	static void
8329	mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev)
8330	{
8331	struct mlxsw_sp_router_hwstats_notify_work *hws_work;
8332
8333	/* To collect notification payload, the core ends up sending another
8334	* notifier block message, which would deadlock on the attempt to
8335	* acquire the router lock again. Just postpone the notification until
8336	* later.
8337	*/
8338
8339	hws_work = kzalloc(size: sizeof(*hws_work), GFP_KERNEL);
8340	if (!hws_work)
8341	return;
8342
8343	INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work);
8344	netdev_hold(dev, tracker: &hws_work->dev_tracker, GFP_KERNEL);
8345	hws_work->dev = dev;
8346	mlxsw_core_schedule_work(work: &hws_work->work);
8347	}
8348
8349	int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
8350	{
8351	return mlxsw_sp_rif_dev(rif)->ifindex;
8352	}
8353
8354	bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif)
8355	{
8356	return !!mlxsw_sp_rif_dev(rif);
8357	}
8358
8359	bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif,
8360	const struct net_device *dev)
8361	{
8362	return mlxsw_sp_rif_dev(rif) == dev;
8363	}
8364
8365	static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif)
8366	{
8367	struct rtnl_hw_stats64 stats = {};
8368
8369	if (!mlxsw_sp_router_port_l3_stats_fetch(rif, p_stats: &stats))
8370	netdev_offload_xstats_push_delta(dev: mlxsw_sp_rif_dev(rif),
8371	type: NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8372	stats: &stats);
8373	}
8374
8375	static struct mlxsw_sp_rif *
8376	mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
8377	const struct mlxsw_sp_rif_params *params,
8378	struct netlink_ext_ack *extack)
8379	{
8380	u8 rif_entries = params->double_entry ? 2 : 1;
8381	u32 tb_id = l3mdev_fib_table(dev: params->dev);
8382	const struct mlxsw_sp_rif_ops *ops;
8383	struct mlxsw_sp_fid *fid = NULL;
8384	enum mlxsw_sp_rif_type type;
8385	struct mlxsw_sp_crif *crif;
8386	struct mlxsw_sp_rif *rif;
8387	struct mlxsw_sp_vr *vr;
8388	u16 rif_index;
8389	int i, err;
8390
8391	type = mlxsw_sp_dev_rif_type(mlxsw_sp, dev: params->dev);
8392	ops = mlxsw_sp->router->rif_ops_arr[type];
8393
8394	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id: tb_id ? : RT_TABLE_MAIN, extack);
8395	if (IS_ERR(ptr: vr))
8396	return ERR_CAST(ptr: vr);
8397	vr->rif_count++;
8398
8399	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, p_rif_index: &rif_index, rif_entries);
8400	if (err) {
8401	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8402	goto err_rif_index_alloc;
8403	}
8404
8405	crif = mlxsw_sp_crif_lookup(router: mlxsw_sp->router, dev: params->dev);
8406	if (WARN_ON(!crif)) {
8407	err = -ENOENT;
8408	goto err_crif_lookup;
8409	}
8410
8411	rif = mlxsw_sp_rif_alloc(rif_size: ops->rif_size, rif_index, vr_id: vr->id, crif);
8412	if (!rif) {
8413	err = -ENOMEM;
8414	goto err_rif_alloc;
8415	}
8416	netdev_hold(dev: params->dev, tracker: &rif->dev_tracker, GFP_KERNEL);
8417	mlxsw_sp->router->rifs[rif_index] = rif;
8418	rif->mlxsw_sp = mlxsw_sp;
8419	rif->ops = ops;
8420	rif->rif_entries = rif_entries;
8421
8422	if (ops->fid_get) {
8423	fid = ops->fid_get(rif, params, extack);
8424	if (IS_ERR(ptr: fid)) {
8425	err = PTR_ERR(ptr: fid);
8426	goto err_fid_get;
8427	}
8428	rif->fid = fid;
8429	}
8430
8431	if (ops->setup)
8432	ops->setup(rif, params);
8433
8434	err = ops->configure(rif, extack);
8435	if (err)
8436	goto err_configure;
8437
8438	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8439	err = mlxsw_sp_mr_rif_add(mr_table: vr->mr_table[i], rif);
8440	if (err)
8441	goto err_mr_rif_add;
8442	}
8443
8444	err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif);
8445	if (err)
8446	goto err_rif_made_sync;
8447
8448	if (netdev_offload_xstats_enabled(dev: params->dev,
8449	type: NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8450	err = mlxsw_sp_router_port_l3_stats_enable(rif);
8451	if (err)
8452	goto err_stats_enable;
8453	mlxsw_sp_router_hwstats_notify_schedule(dev: params->dev);
8454	} else {
8455	mlxsw_sp_rif_counters_alloc(rif);
8456	}
8457
8458	atomic_add(i: rif_entries, v: &mlxsw_sp->router->rifs_count);
8459	return rif;
8460
8461	err_stats_enable:
8462	mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8463	err_rif_made_sync:
8464	err_mr_rif_add:
8465	for (i--; i >= 0; i--)
8466	mlxsw_sp_mr_rif_del(mr_table: vr->mr_table[i], rif);
8467	ops->deconfigure(rif);
8468	err_configure:
8469	if (fid)
8470	mlxsw_sp_fid_put(fid);
8471	err_fid_get:
8472	mlxsw_sp->router->rifs[rif_index] = NULL;
8473	netdev_put(dev: params->dev, tracker: &rif->dev_tracker);
8474	mlxsw_sp_rif_free(rif);
8475	err_rif_alloc:
8476	err_crif_lookup:
8477	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8478	err_rif_index_alloc:
8479	vr->rif_count--;
8480	mlxsw_sp_vr_put(mlxsw_sp, vr);
8481	return ERR_PTR(error: err);
8482	}
8483
8484	static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
8485	{
8486	struct net_device *dev = mlxsw_sp_rif_dev(rif);
8487	const struct mlxsw_sp_rif_ops *ops = rif->ops;
8488	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8489	struct mlxsw_sp_crif *crif = rif->crif;
8490	struct mlxsw_sp_fid *fid = rif->fid;
8491	u8 rif_entries = rif->rif_entries;
8492	u16 rif_index = rif->rif_index;
8493	struct mlxsw_sp_vr *vr;
8494	int i;
8495
8496	atomic_sub(i: rif_entries, v: &mlxsw_sp->router->rifs_count);
8497	mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8498	vr = &mlxsw_sp->router->vrs[rif->vr_id];
8499
8500	if (netdev_offload_xstats_enabled(dev, type: NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8501	mlxsw_sp_rif_push_l3_stats(rif);
8502	mlxsw_sp_router_port_l3_stats_disable(rif);
8503	mlxsw_sp_router_hwstats_notify_schedule(dev);
8504	} else {
8505	mlxsw_sp_rif_counters_free(rif);
8506	}
8507
8508	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
8509	mlxsw_sp_mr_rif_del(mr_table: vr->mr_table[i], rif);
8510	ops->deconfigure(rif);
8511	if (fid)
8512	/* Loopback RIFs are not associated with a FID. */
8513	mlxsw_sp_fid_put(fid);
8514	mlxsw_sp->router->rifs[rif->rif_index] = NULL;
8515	netdev_put(dev, tracker: &rif->dev_tracker);
8516	mlxsw_sp_rif_free(rif);
8517	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8518	vr->rif_count--;
8519	mlxsw_sp_vr_put(mlxsw_sp, vr);
8520
8521	if (crif->can_destroy)
8522	mlxsw_sp_crif_free(crif);
8523	}
8524
8525	void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
8526	struct net_device *dev)
8527	{
8528	struct mlxsw_sp_rif *rif;
8529
8530	mutex_lock(&mlxsw_sp->router->lock);
8531	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8532	if (!rif)
8533	goto out;
8534	mlxsw_sp_rif_destroy(rif);
8535	out:
8536	mutex_unlock(lock: &mlxsw_sp->router->lock);
8537	}
8538
8539	static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp,
8540	struct net_device *br_dev,
8541	u16 vid)
8542	{
8543	struct net_device *upper_dev;
8544	struct mlxsw_sp_crif *crif;
8545
8546	rcu_read_lock();
8547	upper_dev = __vlan_find_dev_deep_rcu(real_dev: br_dev, htons(ETH_P_8021Q), vlan_id: vid);
8548	rcu_read_unlock();
8549
8550	if (!upper_dev)
8551	return;
8552
8553	crif = mlxsw_sp_crif_lookup(router: mlxsw_sp->router, dev: upper_dev);
8554	if (!crif || !crif->rif)
8555	return;
8556
8557	mlxsw_sp_rif_destroy(rif: crif->rif);
8558	}
8559
8560	static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
8561	struct net_device *l3_dev,
8562	int lower_pvid,
8563	unsigned long event,
8564	struct netlink_ext_ack *extack);
8565
8566	int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp,
8567	struct net_device *br_dev,
8568	u16 new_vid, bool is_pvid,
8569	struct netlink_ext_ack *extack)
8570	{
8571	struct mlxsw_sp_rif *old_rif;
8572	struct mlxsw_sp_rif *new_rif;
8573	struct net_device *upper_dev;
8574	u16 old_pvid = 0;
8575	u16 new_pvid;
8576	int err = 0;
8577
8578	mutex_lock(&mlxsw_sp->router->lock);
8579	old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: br_dev);
8580	if (old_rif) {
8581	/* If the RIF on the bridge is not a VLAN RIF, we shouldn't have
8582	* gotten a PVID notification.
8583	*/
8584	if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN))
8585	old_rif = NULL;
8586	else
8587	old_pvid = mlxsw_sp_fid_8021q_vid(fid: old_rif->fid);
8588	}
8589
8590	if (is_pvid)
8591	new_pvid = new_vid;
8592	else if (old_pvid == new_vid)
8593	new_pvid = 0;
8594	else
8595	goto out;
8596
8597	if (old_pvid == new_pvid)
8598	goto out;
8599
8600	if (new_pvid) {
8601	struct mlxsw_sp_rif_params params = {
8602	.dev = br_dev,
8603	.vid = new_pvid,
8604	};
8605
8606	/* If there is a VLAN upper with the same VID as the new PVID,
8607	* kill its RIF, if there is one.
8608	*/
8609	mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, vid: new_pvid);
8610
8611	if (mlxsw_sp_dev_addr_list_empty(dev: br_dev))
8612	goto out;
8613	new_rif = mlxsw_sp_rif_create(mlxsw_sp, params: &params, extack);
8614	if (IS_ERR(ptr: new_rif)) {
8615	err = PTR_ERR(ptr: new_rif);
8616	goto out;
8617	}
8618
8619	if (old_pvid)
8620	mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif,
8621	migrate_nhs: true);
8622	} else {
8623	mlxsw_sp_rif_destroy(rif: old_rif);
8624	}
8625
8626	if (old_pvid) {
8627	rcu_read_lock();
8628	upper_dev = __vlan_find_dev_deep_rcu(real_dev: br_dev, htons(ETH_P_8021Q),
8629	vlan_id: old_pvid);
8630	rcu_read_unlock();
8631	if (upper_dev)
8632	err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp,
8633	l3_dev: upper_dev,
8634	lower_pvid: new_pvid,
8635	event: NETDEV_UP, extack);
8636	}
8637
8638	out:
8639	mutex_unlock(lock: &mlxsw_sp->router->lock);
8640	return err;
8641	}
8642
8643	static void
8644	mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
8645	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8646	{
8647	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8648
8649	params->vid = mlxsw_sp_port_vlan->vid;
8650	params->lag = mlxsw_sp_port->lagged;
8651	if (params->lag)
8652	params->lag_id = mlxsw_sp_port->lag_id;
8653	else
8654	params->system_port = mlxsw_sp_port->local_port;
8655	}
8656
8657	static struct mlxsw_sp_rif_subport *
8658	mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
8659	{
8660	return container_of(rif, struct mlxsw_sp_rif_subport, common);
8661	}
8662
8663	static struct mlxsw_sp_rif *
8664	mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
8665	const struct mlxsw_sp_rif_params *params,
8666	struct netlink_ext_ack *extack)
8667	{
8668	struct mlxsw_sp_rif_subport *rif_subport;
8669	struct mlxsw_sp_rif *rif;
8670
8671	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: params->dev);
8672	if (!rif)
8673	return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
8674
8675	rif_subport = mlxsw_sp_rif_subport_rif(rif);
8676	refcount_inc(r: &rif_subport->ref_count);
8677	return rif;
8678	}
8679
8680	static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
8681	{
8682	struct mlxsw_sp_rif_subport *rif_subport;
8683
8684	rif_subport = mlxsw_sp_rif_subport_rif(rif);
8685	if (!refcount_dec_and_test(r: &rif_subport->ref_count))
8686	return;
8687
8688	mlxsw_sp_rif_destroy(rif);
8689	}
8690
8691	static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp,
8692	struct mlxsw_sp_rif_mac_profile *profile,
8693	struct netlink_ext_ack *extack)
8694	{
8695	u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
8696	struct mlxsw_sp_router *router = mlxsw_sp->router;
8697	int id;
8698
8699	id = idr_alloc(&router->rif_mac_profiles_idr, ptr: profile, start: 0,
8700	end: max_rif_mac_profiles, GFP_KERNEL);
8701
8702	if (id >= 0) {
8703	profile->id = id;
8704	return 0;
8705	}
8706
8707	if (id == -ENOSPC)
8708	NL_SET_ERR_MSG_MOD(extack,
8709	"Exceeded number of supported router interface MAC profiles");
8710
8711	return id;
8712	}
8713
8714	static struct mlxsw_sp_rif_mac_profile *
8715	mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile)
8716	{
8717	struct mlxsw_sp_rif_mac_profile *profile;
8718
8719	profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr,
8720	id: mac_profile);
8721	WARN_ON(!profile);
8722	return profile;
8723	}
8724
8725	static struct mlxsw_sp_rif_mac_profile *
8726	mlxsw_sp_rif_mac_profile_alloc(const char *mac)
8727	{
8728	struct mlxsw_sp_rif_mac_profile *profile;
8729
8730	profile = kzalloc(size: sizeof(*profile), GFP_KERNEL);
8731	if (!profile)
8732	return NULL;
8733
8734	ether_addr_copy(dst: profile->mac_prefix, src: mac);
8735	refcount_set(r: &profile->ref_count, n: 1);
8736	return profile;
8737	}
8738
8739	static struct mlxsw_sp_rif_mac_profile *
8740	mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac)
8741	{
8742	struct mlxsw_sp_router *router = mlxsw_sp->router;
8743	struct mlxsw_sp_rif_mac_profile *profile;
8744	int id;
8745
8746	idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) {
8747	if (ether_addr_equal_masked(addr1: profile->mac_prefix, addr2: mac,
8748	mask: mlxsw_sp->mac_mask))
8749	return profile;
8750	}
8751
8752	return NULL;
8753	}
8754
8755	static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv)
8756	{
8757	const struct mlxsw_sp *mlxsw_sp = priv;
8758
8759	return atomic_read(v: &mlxsw_sp->router->rif_mac_profiles_count);
8760	}
8761
8762	static u64 mlxsw_sp_rifs_occ_get(void *priv)
8763	{
8764	const struct mlxsw_sp *mlxsw_sp = priv;
8765
8766	return atomic_read(v: &mlxsw_sp->router->rifs_count);
8767	}
8768
8769	static struct mlxsw_sp_rif_mac_profile *
8770	mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac,
8771	struct netlink_ext_ack *extack)
8772	{
8773	struct mlxsw_sp_rif_mac_profile *profile;
8774	int err;
8775
8776	profile = mlxsw_sp_rif_mac_profile_alloc(mac);
8777	if (!profile)
8778	return ERR_PTR(error: -ENOMEM);
8779
8780	err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack);
8781	if (err)
8782	goto profile_index_alloc_err;
8783
8784	atomic_inc(v: &mlxsw_sp->router->rif_mac_profiles_count);
8785	return profile;
8786
8787	profile_index_alloc_err:
8788	kfree(objp: profile);
8789	return ERR_PTR(error: err);
8790	}
8791
8792	static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp,
8793	u8 mac_profile)
8794	{
8795	struct mlxsw_sp_rif_mac_profile *profile;
8796
8797	atomic_dec(v: &mlxsw_sp->router->rif_mac_profiles_count);
8798	profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile);
8799	kfree(objp: profile);
8800	}
8801
8802	static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp,
8803	const char *mac, u8 *p_mac_profile,
8804	struct netlink_ext_ack *extack)
8805	{
8806	struct mlxsw_sp_rif_mac_profile *profile;
8807
8808	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac);
8809	if (profile) {
8810	refcount_inc(r: &profile->ref_count);
8811	goto out;
8812	}
8813
8814	profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack);
8815	if (IS_ERR(ptr: profile))
8816	return PTR_ERR(ptr: profile);
8817
8818	out:
8819	*p_mac_profile = profile->id;
8820	return 0;
8821	}
8822
8823	static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp,
8824	u8 mac_profile)
8825	{
8826	struct mlxsw_sp_rif_mac_profile *profile;
8827
8828	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8829	id: mac_profile);
8830	if (WARN_ON(!profile))
8831	return;
8832
8833	if (!refcount_dec_and_test(r: &profile->ref_count))
8834	return;
8835
8836	mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile);
8837	}
8838
8839	static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif)
8840	{
8841	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8842	struct mlxsw_sp_rif_mac_profile *profile;
8843
8844	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8845	id: rif->mac_profile_id);
8846	if (WARN_ON(!profile))
8847	return false;
8848
8849	return refcount_read(r: &profile->ref_count) > 1;
8850	}
8851
8852	static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif,
8853	const char *new_mac)
8854	{
8855	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8856	struct mlxsw_sp_rif_mac_profile *profile;
8857
8858	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8859	id: rif->mac_profile_id);
8860	if (WARN_ON(!profile))
8861	return -EINVAL;
8862
8863	ether_addr_copy(dst: profile->mac_prefix, src: new_mac);
8864	return 0;
8865	}
8866
8867	static int
8868	mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp,
8869	struct mlxsw_sp_rif *rif,
8870	const char *new_mac,
8871	struct netlink_ext_ack *extack)
8872	{
8873	u8 mac_profile;
8874	int err;
8875
8876	if (!mlxsw_sp_rif_mac_profile_is_shared(rif) &&
8877	!mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac: new_mac))
8878	return mlxsw_sp_rif_mac_profile_edit(rif, new_mac);
8879
8880	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, mac: new_mac,
8881	p_mac_profile: &mac_profile, extack);
8882	if (err)
8883	return err;
8884
8885	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile: rif->mac_profile_id);
8886	rif->mac_profile_id = mac_profile;
8887	return 0;
8888	}
8889
8890	static int
8891	__mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8892	struct net_device *l3_dev,
8893	struct netlink_ext_ack *extack)
8894	{
8895	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8896	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
8897	struct mlxsw_sp_rif_params params;
8898	u16 vid = mlxsw_sp_port_vlan->vid;
8899	struct mlxsw_sp_rif *rif;
8900	struct mlxsw_sp_fid *fid;
8901	int err;
8902
8903	params = (struct mlxsw_sp_rif_params) {
8904	.dev = l3_dev,
8905	.vid = vid,
8906	};
8907
8908	mlxsw_sp_rif_subport_params_init(params: &params, mlxsw_sp_port_vlan);
8909	rif = mlxsw_sp_rif_subport_get(mlxsw_sp, params: &params, extack);
8910	if (IS_ERR(ptr: rif))
8911	return PTR_ERR(ptr: rif);
8912
8913	/* FID was already created, just take a reference */
8914	fid = rif->ops->fid_get(rif, &params, extack);
8915	err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
8916	if (err)
8917	goto err_fid_port_vid_map;
8918
8919	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, learn_enable: false);
8920	if (err)
8921	goto err_port_vid_learning_set;
8922
8923	err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
8924	BR_STATE_FORWARDING);
8925	if (err)
8926	goto err_port_vid_stp_set;
8927
8928	mlxsw_sp_port_vlan->fid = fid;
8929
8930	return 0;
8931
8932	err_port_vid_stp_set:
8933	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, learn_enable: true);
8934	err_port_vid_learning_set:
8935	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8936	err_fid_port_vid_map:
8937	mlxsw_sp_fid_put(fid);
8938	mlxsw_sp_rif_subport_put(rif);
8939	return err;
8940	}
8941
8942	static void
8943	__mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8944	{
8945	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8946	struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
8947	struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
8948	u16 vid = mlxsw_sp_port_vlan->vid;
8949
8950	if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
8951	return;
8952
8953	mlxsw_sp_port_vlan->fid = NULL;
8954	mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
8955	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, learn_enable: true);
8956	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8957	mlxsw_sp_fid_put(fid);
8958	mlxsw_sp_rif_subport_put(rif);
8959	}
8960
8961	static int
8962	mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8963	struct net_device *l3_dev,
8964	struct netlink_ext_ack *extack)
8965	{
8966	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8967
8968	lockdep_assert_held(&mlxsw_sp->router->lock);
8969
8970	if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: l3_dev))
8971	return 0;
8972
8973	return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev,
8974	extack);
8975	}
8976
8977	void
8978	mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8979	{
8980	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8981
8982	mutex_lock(&mlxsw_sp->router->lock);
8983	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
8984	mutex_unlock(lock: &mlxsw_sp->router->lock);
8985	}
8986
8987	static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
8988	struct net_device *port_dev,
8989	unsigned long event, u16 vid,
8990	struct netlink_ext_ack *extack)
8991	{
8992	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev: port_dev);
8993	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
8994
8995	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
8996	if (WARN_ON(!mlxsw_sp_port_vlan))
8997	return -EINVAL;
8998
8999	switch (event) {
9000	case NETDEV_UP:
9001	return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
9002	l3_dev, extack);
9003	case NETDEV_DOWN:
9004	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
9005	break;
9006	}
9007
9008	return 0;
9009	}
9010
9011	static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
9012	unsigned long event, bool nomaster,
9013	struct netlink_ext_ack *extack)
9014	{
9015	if (!nomaster && (netif_is_any_bridge_port(dev: port_dev) ||
9016	netif_is_lag_port(dev: port_dev)))
9017	return 0;
9018
9019	return mlxsw_sp_inetaddr_port_vlan_event(l3_dev: port_dev, port_dev, event,
9020	MLXSW_SP_DEFAULT_VID, extack);
9021	}
9022
9023	static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
9024	struct net_device *lag_dev,
9025	unsigned long event, u16 vid,
9026	struct netlink_ext_ack *extack)
9027	{
9028	struct net_device *port_dev;
9029	struct list_head *iter;
9030	int err;
9031
9032	netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
9033	if (mlxsw_sp_port_dev_check(dev: port_dev)) {
9034	err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
9035	port_dev,
9036	event, vid,
9037	extack);
9038	if (err)
9039	return err;
9040	}
9041	}
9042
9043	return 0;
9044	}
9045
9046	static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
9047	unsigned long event, bool nomaster,
9048	struct netlink_ext_ack *extack)
9049	{
9050	if (!nomaster && netif_is_bridge_port(dev: lag_dev))
9051	return 0;
9052
9053	return __mlxsw_sp_inetaddr_lag_event(l3_dev: lag_dev, lag_dev, event,
9054	MLXSW_SP_DEFAULT_VID, extack);
9055	}
9056
9057	static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
9058	struct net_device *l3_dev,
9059	int lower_pvid,
9060	unsigned long event,
9061	struct netlink_ext_ack *extack)
9062	{
9063	struct mlxsw_sp_rif_params params = {
9064	.dev = l3_dev,
9065	};
9066	struct mlxsw_sp_rif *rif;
9067	int err;
9068
9069	switch (event) {
9070	case NETDEV_UP:
9071	if (netif_is_bridge_master(dev: l3_dev) && br_vlan_enabled(dev: l3_dev)) {
9072	u16 proto;
9073
9074	br_vlan_get_proto(dev: l3_dev, p_proto: &proto);
9075	if (proto == ETH_P_8021AD) {
9076	NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported");
9077	return -EOPNOTSUPP;
9078	}
9079	err = br_vlan_get_pvid(dev: l3_dev, p_pvid: &params.vid);
9080	if (err)
9081	return err;
9082	if (!params.vid)
9083	return 0;
9084	} else if (is_vlan_dev(dev: l3_dev)) {
9085	params.vid = vlan_dev_vlan_id(dev: l3_dev);
9086
9087	/* If the VID matches PVID of the bridge below, the
9088	* bridge owns the RIF for this VLAN. Don't do anything.
9089	*/
9090	if ((int)params.vid == lower_pvid)
9091	return 0;
9092	}
9093
9094	rif = mlxsw_sp_rif_create(mlxsw_sp, params: &params, extack);
9095	if (IS_ERR(ptr: rif))
9096	return PTR_ERR(ptr: rif);
9097	break;
9098	case NETDEV_DOWN:
9099	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: l3_dev);
9100	mlxsw_sp_rif_destroy(rif);
9101	break;
9102	}
9103
9104	return 0;
9105	}
9106
9107	static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
9108	struct net_device *vlan_dev,
9109	unsigned long event, bool nomaster,
9110	struct netlink_ext_ack *extack)
9111	{
9112	struct net_device *real_dev = vlan_dev_real_dev(dev: vlan_dev);
9113	u16 vid = vlan_dev_vlan_id(dev: vlan_dev);
9114	u16 lower_pvid;
9115	int err;
9116
9117	if (!nomaster && netif_is_bridge_port(dev: vlan_dev))
9118	return 0;
9119
9120	if (mlxsw_sp_port_dev_check(dev: real_dev)) {
9121	return mlxsw_sp_inetaddr_port_vlan_event(l3_dev: vlan_dev, port_dev: real_dev,
9122	event, vid, extack);
9123	} else if (netif_is_lag_master(dev: real_dev)) {
9124	return __mlxsw_sp_inetaddr_lag_event(l3_dev: vlan_dev, lag_dev: real_dev, event,
9125	vid, extack);
9126	} else if (netif_is_bridge_master(dev: real_dev) &&
9127	br_vlan_enabled(dev: real_dev)) {
9128	err = br_vlan_get_pvid(dev: real_dev, p_pvid: &lower_pvid);
9129	if (err)
9130	return err;
9131	return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, l3_dev: vlan_dev,
9132	lower_pvid, event,
9133	extack);
9134	}
9135
9136	return 0;
9137	}
9138
9139	static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
9140	{
9141	u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
9142	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
9143
9144	return ether_addr_equal_masked(addr1: mac, addr2: vrrp4, mask);
9145	}
9146
9147	static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
9148	{
9149	u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
9150	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
9151
9152	return ether_addr_equal_masked(addr1: mac, addr2: vrrp6, mask);
9153	}
9154
9155	static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
9156	const u8 *mac, bool adding)
9157	{
9158	char ritr_pl[MLXSW_REG_RITR_LEN];
9159	u8 vrrp_id = adding ? mac[5] : 0;
9160	int err;
9161
9162	if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
9163	!mlxsw_sp_rif_macvlan_is_vrrp6(mac))
9164	return 0;
9165
9166	mlxsw_reg_ritr_rif_pack(payload: ritr_pl, rif: rif_index);
9167	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
9168	if (err)
9169	return err;
9170
9171	if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
9172	mlxsw_reg_ritr_if_vrrp_id_ipv4_set(buf: ritr_pl, val: vrrp_id);
9173	else
9174	mlxsw_reg_ritr_if_vrrp_id_ipv6_set(buf: ritr_pl, val: vrrp_id);
9175
9176	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
9177	}
9178
9179	static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
9180	const struct net_device *macvlan_dev,
9181	struct netlink_ext_ack *extack)
9182	{
9183	struct macvlan_dev *vlan = netdev_priv(dev: macvlan_dev);
9184	struct mlxsw_sp_rif *rif;
9185	int err;
9186
9187	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: vlan->lowerdev);
9188	if (!rif)
9189	return 0;
9190
9191	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, mac: macvlan_dev->dev_addr,
9192	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: true);
9193	if (err)
9194	return err;
9195
9196	err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif_index: rif->rif_index,
9197	mac: macvlan_dev->dev_addr, adding: true);
9198	if (err)
9199	goto err_rif_vrrp_add;
9200
9201	/* Make sure the bridge driver does not have this MAC pointing at
9202	* some other port.
9203	*/
9204	if (rif->ops->fdb_del)
9205	rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
9206
9207	return 0;
9208
9209	err_rif_vrrp_add:
9210	mlxsw_sp_rif_fdb_op(mlxsw_sp, mac: macvlan_dev->dev_addr,
9211	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
9212	return err;
9213	}
9214
9215	static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
9216	const struct net_device *macvlan_dev)
9217	{
9218	struct macvlan_dev *vlan = netdev_priv(dev: macvlan_dev);
9219	struct mlxsw_sp_rif *rif;
9220
9221	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: vlan->lowerdev);
9222	/* If we do not have a RIF, then we already took care of
9223	* removing the macvlan's MAC during RIF deletion.
9224	*/
9225	if (!rif)
9226	return;
9227	mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif_index: rif->rif_index, mac: macvlan_dev->dev_addr,
9228	adding: false);
9229	mlxsw_sp_rif_fdb_op(mlxsw_sp, mac: macvlan_dev->dev_addr,
9230	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
9231	}
9232
9233	void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
9234	const struct net_device *macvlan_dev)
9235	{
9236	mutex_lock(&mlxsw_sp->router->lock);
9237	__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
9238	mutex_unlock(lock: &mlxsw_sp->router->lock);
9239	}
9240
9241	static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
9242	struct net_device *macvlan_dev,
9243	unsigned long event,
9244	struct netlink_ext_ack *extack)
9245	{
9246	switch (event) {
9247	case NETDEV_UP:
9248	return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
9249	case NETDEV_DOWN:
9250	__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
9251	break;
9252	}
9253
9254	return 0;
9255	}
9256
9257	static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
9258	struct net_device *dev,
9259	unsigned long event, bool nomaster,
9260	struct netlink_ext_ack *extack)
9261	{
9262	if (mlxsw_sp_port_dev_check(dev))
9263	return mlxsw_sp_inetaddr_port_event(port_dev: dev, event, nomaster,
9264	extack);
9265	else if (netif_is_lag_master(dev))
9266	return mlxsw_sp_inetaddr_lag_event(lag_dev: dev, event, nomaster,
9267	extack);
9268	else if (netif_is_bridge_master(dev))
9269	return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, l3_dev: dev, lower_pvid: -1, event,
9270	extack);
9271	else if (is_vlan_dev(dev))
9272	return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, vlan_dev: dev, event,
9273	nomaster, extack);
9274	else if (netif_is_macvlan(dev))
9275	return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, macvlan_dev: dev, event,
9276	extack);
9277	else
9278	return 0;
9279	}
9280
9281	static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
9282	unsigned long event, void *ptr)
9283	{
9284	struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
9285	struct net_device *dev = ifa->ifa_dev->dev;
9286	struct mlxsw_sp_router *router;
9287	struct mlxsw_sp_rif *rif;
9288	int err = 0;
9289
9290	/* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
9291	if (event == NETDEV_UP)
9292	return NOTIFY_DONE;
9293
9294	router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
9295	mutex_lock(&router->lock);
9296	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp: router->mlxsw_sp, dev);
9297	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9298	goto out;
9299
9300	err = __mlxsw_sp_inetaddr_event(mlxsw_sp: router->mlxsw_sp, dev, event, nomaster: false,
9301	NULL);
9302	out:
9303	mutex_unlock(lock: &router->lock);
9304	return notifier_from_errno(err);
9305	}
9306
9307	static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
9308	unsigned long event, void *ptr)
9309	{
9310	struct in_validator_info *ivi = (struct in_validator_info *) ptr;
9311	struct net_device *dev = ivi->ivi_dev->dev;
9312	struct mlxsw_sp *mlxsw_sp;
9313	struct mlxsw_sp_rif *rif;
9314	int err = 0;
9315
9316	mlxsw_sp = mlxsw_sp_lower_get(dev);
9317	if (!mlxsw_sp)
9318	return NOTIFY_DONE;
9319
9320	mutex_lock(&mlxsw_sp->router->lock);
9321	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9322	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9323	goto out;
9324
9325	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, nomaster: false,
9326	extack: ivi->extack);
9327	out:
9328	mutex_unlock(lock: &mlxsw_sp->router->lock);
9329	return notifier_from_errno(err);
9330	}
9331
9332	struct mlxsw_sp_inet6addr_event_work {
9333	struct work_struct work;
9334	struct mlxsw_sp *mlxsw_sp;
9335	struct net_device *dev;
9336	netdevice_tracker dev_tracker;
9337	unsigned long event;
9338	};
9339
9340	static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
9341	{
9342	struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
9343	container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
9344	struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
9345	struct net_device *dev = inet6addr_work->dev;
9346	unsigned long event = inet6addr_work->event;
9347	struct mlxsw_sp_rif *rif;
9348
9349	rtnl_lock();
9350	mutex_lock(&mlxsw_sp->router->lock);
9351
9352	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9353	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9354	goto out;
9355
9356	__mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, nomaster: false, NULL);
9357	out:
9358	mutex_unlock(lock: &mlxsw_sp->router->lock);
9359	rtnl_unlock();
9360	netdev_put(dev, tracker: &inet6addr_work->dev_tracker);
9361	kfree(objp: inet6addr_work);
9362	}
9363
9364	/* Called with rcu_read_lock() */
9365	static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
9366	unsigned long event, void *ptr)
9367	{
9368	struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
9369	struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
9370	struct net_device *dev = if6->idev->dev;
9371	struct mlxsw_sp_router *router;
9372
9373	/* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
9374	if (event == NETDEV_UP)
9375	return NOTIFY_DONE;
9376
9377	inet6addr_work = kzalloc(size: sizeof(*inet6addr_work), GFP_ATOMIC);
9378	if (!inet6addr_work)
9379	return NOTIFY_BAD;
9380
9381	router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
9382	INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
9383	inet6addr_work->mlxsw_sp = router->mlxsw_sp;
9384	inet6addr_work->dev = dev;
9385	inet6addr_work->event = event;
9386	netdev_hold(dev, tracker: &inet6addr_work->dev_tracker, GFP_ATOMIC);
9387	mlxsw_core_schedule_work(work: &inet6addr_work->work);
9388
9389	return NOTIFY_DONE;
9390	}
9391
9392	static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
9393	unsigned long event, void *ptr)
9394	{
9395	struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
9396	struct net_device *dev = i6vi->i6vi_dev->dev;
9397	struct mlxsw_sp *mlxsw_sp;
9398	struct mlxsw_sp_rif *rif;
9399	int err = 0;
9400
9401	mlxsw_sp = mlxsw_sp_lower_get(dev);
9402	if (!mlxsw_sp)
9403	return NOTIFY_DONE;
9404
9405	mutex_lock(&mlxsw_sp->router->lock);
9406	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9407	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9408	goto out;
9409
9410	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, nomaster: false,
9411	extack: i6vi->extack);
9412	out:
9413	mutex_unlock(lock: &mlxsw_sp->router->lock);
9414	return notifier_from_errno(err);
9415	}
9416
9417	static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
9418	const char *mac, int mtu, u8 mac_profile)
9419	{
9420	char ritr_pl[MLXSW_REG_RITR_LEN];
9421	int err;
9422
9423	mlxsw_reg_ritr_rif_pack(payload: ritr_pl, rif: rif_index);
9424	err = mlxsw_reg_query(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
9425	if (err)
9426	return err;
9427
9428	mlxsw_reg_ritr_mtu_set(buf: ritr_pl, val: mtu);
9429	mlxsw_reg_ritr_if_mac_memcpy_to(buf: ritr_pl, src: mac);
9430	mlxsw_reg_ritr_if_mac_profile_id_set(buf: ritr_pl, val: mac_profile);
9431	mlxsw_reg_ritr_op_set(buf: ritr_pl, val: MLXSW_REG_RITR_RIF_CREATE);
9432	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
9433	}
9434
9435	static int
9436	mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
9437	struct mlxsw_sp_rif *rif,
9438	struct netlink_ext_ack *extack)
9439	{
9440	struct net_device *dev = mlxsw_sp_rif_dev(rif);
9441	u8 old_mac_profile;
9442	u16 fid_index;
9443	int err;
9444
9445	fid_index = mlxsw_sp_fid_index(fid: rif->fid);
9446
9447	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, mac: rif->addr, fid: fid_index, adding: false);
9448	if (err)
9449	return err;
9450
9451	old_mac_profile = rif->mac_profile_id;
9452	err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, new_mac: dev->dev_addr,
9453	extack);
9454	if (err)
9455	goto err_rif_mac_profile_replace;
9456
9457	err = mlxsw_sp_rif_edit(mlxsw_sp, rif_index: rif->rif_index, mac: dev->dev_addr,
9458	mtu: dev->mtu, mac_profile: rif->mac_profile_id);
9459	if (err)
9460	goto err_rif_edit;
9461
9462	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, mac: dev->dev_addr, fid: fid_index, adding: true);
9463	if (err)
9464	goto err_rif_fdb_op;
9465
9466	if (rif->mtu != dev->mtu) {
9467	struct mlxsw_sp_vr *vr;
9468	int i;
9469
9470	/* The RIF is relevant only to its mr_table instance, as unlike
9471	* unicast routing, in multicast routing a RIF cannot be shared
9472	* between several multicast routing tables.
9473	*/
9474	vr = &mlxsw_sp->router->vrs[rif->vr_id];
9475	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
9476	mlxsw_sp_mr_rif_mtu_update(mr_table: vr->mr_table[i],
9477	rif, mtu: dev->mtu);
9478	}
9479
9480	ether_addr_copy(dst: rif->addr, src: dev->dev_addr);
9481	rif->mtu = dev->mtu;
9482
9483	netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
9484
9485	return 0;
9486
9487	err_rif_fdb_op:
9488	mlxsw_sp_rif_edit(mlxsw_sp, rif_index: rif->rif_index, mac: rif->addr, mtu: rif->mtu,
9489	mac_profile: old_mac_profile);
9490	err_rif_edit:
9491	mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, new_mac: rif->addr, extack);
9492	err_rif_mac_profile_replace:
9493	mlxsw_sp_rif_fdb_op(mlxsw_sp, mac: rif->addr, fid: fid_index, adding: true);
9494	return err;
9495	}
9496
9497	static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
9498	struct netdev_notifier_pre_changeaddr_info *info)
9499	{
9500	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9501	struct mlxsw_sp_rif_mac_profile *profile;
9502	struct netlink_ext_ack *extack;
9503	u8 max_rif_mac_profiles;
9504	u64 occ;
9505
9506	extack = netdev_notifier_info_to_extack(info: &info->info);
9507
9508	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac: info->dev_addr);
9509	if (profile)
9510	return 0;
9511
9512	max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
9513	occ = mlxsw_sp_rif_mac_profiles_occ_get(priv: mlxsw_sp);
9514	if (occ < max_rif_mac_profiles)
9515	return 0;
9516
9517	if (!mlxsw_sp_rif_mac_profile_is_shared(rif))
9518	return 0;
9519
9520	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles");
9521	return -ENOBUFS;
9522	}
9523
9524	static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp,
9525	struct net_device *dev)
9526	{
9527	struct vlan_dev_priv *vlan;
9528
9529	if (netif_is_lag_master(dev) ||
9530	netif_is_bridge_master(dev) ||
9531	mlxsw_sp_port_dev_check(dev) ||
9532	mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) ||
9533	netif_is_l3_master(dev))
9534	return true;
9535
9536	if (!is_vlan_dev(dev))
9537	return false;
9538
9539	vlan = vlan_dev_priv(dev);
9540	return netif_is_lag_master(dev: vlan->real_dev) ||
9541	netif_is_bridge_master(dev: vlan->real_dev) ||
9542	mlxsw_sp_port_dev_check(dev: vlan->real_dev);
9543	}
9544
9545	static struct mlxsw_sp_crif *
9546	mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev)
9547	{
9548	struct mlxsw_sp_crif *crif;
9549	int err;
9550
9551	if (WARN_ON(mlxsw_sp_crif_lookup(router, dev)))
9552	return NULL;
9553
9554	crif = mlxsw_sp_crif_alloc(dev);
9555	if (!crif)
9556	return ERR_PTR(error: -ENOMEM);
9557
9558	err = mlxsw_sp_crif_insert(router, crif);
9559	if (err)
9560	goto err_netdev_insert;
9561
9562	return crif;
9563
9564	err_netdev_insert:
9565	mlxsw_sp_crif_free(crif);
9566	return ERR_PTR(error: err);
9567	}
9568
9569	static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router,
9570	struct mlxsw_sp_crif *crif)
9571	{
9572	struct mlxsw_sp_nexthop *nh, *tmp;
9573
9574	mlxsw_sp_crif_remove(router, crif);
9575
9576	list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node)
9577	mlxsw_sp_nexthop_type_fini(mlxsw_sp: router->mlxsw_sp, nh);
9578
9579	if (crif->rif)
9580	crif->can_destroy = true;
9581	else
9582	mlxsw_sp_crif_free(crif);
9583	}
9584
9585	static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router,
9586	struct net_device *dev)
9587	{
9588	struct mlxsw_sp_crif *crif;
9589
9590	if (!mlxsw_sp_router_netdevice_interesting(mlxsw_sp: router->mlxsw_sp, dev))
9591	return 0;
9592
9593	crif = mlxsw_sp_crif_register(router, dev);
9594	return PTR_ERR_OR_ZERO(ptr: crif);
9595	}
9596
9597	static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router,
9598	struct net_device *dev)
9599	{
9600	struct mlxsw_sp_crif *crif;
9601
9602	if (!mlxsw_sp_router_netdevice_interesting(mlxsw_sp: router->mlxsw_sp, dev))
9603	return;
9604
9605	/* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts
9606	* the NETDEV_UNREGISTER message, so we can get here twice. If that's
9607	* what happened, the netdevice state is NETREG_UNREGISTERED. In that
9608	* case, we expect to have collected the CRIF already, and warn if it
9609	* still exists. Otherwise we expect the CRIF to exist.
9610	*/
9611	crif = mlxsw_sp_crif_lookup(router, dev);
9612	if (dev->reg_state == NETREG_UNREGISTERED) {
9613	if (!WARN_ON(crif))
9614	return;
9615	}
9616	if (WARN_ON(!crif))
9617	return;
9618
9619	mlxsw_sp_crif_unregister(router, crif);
9620	}
9621
9622	static bool mlxsw_sp_is_offload_xstats_event(unsigned long event)
9623	{
9624	switch (event) {
9625	case NETDEV_OFFLOAD_XSTATS_ENABLE:
9626	case NETDEV_OFFLOAD_XSTATS_DISABLE:
9627	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9628	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9629	return true;
9630	}
9631
9632	return false;
9633	}
9634
9635	static int
9636	mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif,
9637	unsigned long event,
9638	struct netdev_notifier_offload_xstats_info *info)
9639	{
9640	switch (info->type) {
9641	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9642	break;
9643	default:
9644	return 0;
9645	}
9646
9647	switch (event) {
9648	case NETDEV_OFFLOAD_XSTATS_ENABLE:
9649	return mlxsw_sp_router_port_l3_stats_enable(rif);
9650	case NETDEV_OFFLOAD_XSTATS_DISABLE:
9651	mlxsw_sp_router_port_l3_stats_disable(rif);
9652	return 0;
9653	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9654	mlxsw_sp_router_port_l3_stats_report_used(rif, info);
9655	return 0;
9656	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9657	return mlxsw_sp_router_port_l3_stats_report_delta(rif, info);
9658	}
9659
9660	WARN_ON_ONCE(1);
9661	return 0;
9662	}
9663
9664	static int
9665	mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp,
9666	struct net_device *dev,
9667	unsigned long event,
9668	struct netdev_notifier_offload_xstats_info *info)
9669	{
9670	struct mlxsw_sp_rif *rif;
9671
9672	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9673	if (!rif)
9674	return 0;
9675
9676	return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info);
9677	}
9678
9679	static bool mlxsw_sp_is_router_event(unsigned long event)
9680	{
9681	switch (event) {
9682	case NETDEV_PRE_CHANGEADDR:
9683	case NETDEV_CHANGEADDR:
9684	case NETDEV_CHANGEMTU:
9685	return true;
9686	default:
9687	return false;
9688	}
9689	}
9690
9691	static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
9692	unsigned long event, void *ptr)
9693	{
9694	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(info: ptr);
9695	struct mlxsw_sp *mlxsw_sp;
9696	struct mlxsw_sp_rif *rif;
9697
9698	mlxsw_sp = mlxsw_sp_lower_get(dev);
9699	if (!mlxsw_sp)
9700	return 0;
9701
9702	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9703	if (!rif)
9704	return 0;
9705
9706	switch (event) {
9707	case NETDEV_CHANGEMTU:
9708	case NETDEV_CHANGEADDR:
9709	return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack);
9710	case NETDEV_PRE_CHANGEADDR:
9711	return mlxsw_sp_router_port_pre_changeaddr_event(rif, info: ptr);
9712	default:
9713	WARN_ON_ONCE(1);
9714	break;
9715	}
9716
9717	return 0;
9718	}
9719
9720	static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
9721	struct net_device *l3_dev,
9722	struct netlink_ext_ack *extack)
9723	{
9724	struct mlxsw_sp_rif *rif;
9725
9726	/* If netdev is already associated with a RIF, then we need to
9727	* destroy it and create a new one with the new virtual router ID.
9728	*/
9729	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: l3_dev);
9730	if (rif)
9731	__mlxsw_sp_inetaddr_event(mlxsw_sp, dev: l3_dev, event: NETDEV_DOWN, nomaster: false,
9732	extack);
9733
9734	return __mlxsw_sp_inetaddr_event(mlxsw_sp, dev: l3_dev, event: NETDEV_UP, nomaster: false,
9735	extack);
9736	}
9737
9738	static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
9739	struct net_device *l3_dev)
9740	{
9741	struct mlxsw_sp_rif *rif;
9742
9743	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev: l3_dev);
9744	if (!rif)
9745	return;
9746	__mlxsw_sp_inetaddr_event(mlxsw_sp, dev: l3_dev, event: NETDEV_DOWN, nomaster: false, NULL);
9747	}
9748
9749	static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr)
9750	{
9751	struct netdev_notifier_changeupper_info *info = ptr;
9752
9753	if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER)
9754	return false;
9755	return netif_is_l3_master(dev: info->upper_dev);
9756	}
9757
9758	static int
9759	mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
9760	struct netdev_notifier_changeupper_info *info)
9761	{
9762	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(dev: l3_dev);
9763	int err = 0;
9764
9765	/* We do not create a RIF for a macvlan, but only use it to
9766	* direct more MAC addresses to the router.
9767	*/
9768	if (!mlxsw_sp || netif_is_macvlan(dev: l3_dev))
9769	return 0;
9770
9771	switch (event) {
9772	case NETDEV_PRECHANGEUPPER:
9773	break;
9774	case NETDEV_CHANGEUPPER:
9775	if (info->linking) {
9776	struct netlink_ext_ack *extack;
9777
9778	extack = netdev_notifier_info_to_extack(info: &info->info);
9779	err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
9780	} else {
9781	mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
9782	}
9783	break;
9784	}
9785
9786	return err;
9787	}
9788
9789	struct mlxsw_sp_router_replay_inetaddr_up {
9790	struct mlxsw_sp *mlxsw_sp;
9791	struct netlink_ext_ack *extack;
9792	unsigned int done;
9793	bool deslavement;
9794	};
9795
9796	static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev,
9797	struct netdev_nested_priv *priv)
9798	{
9799	struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data;
9800	bool nomaster = ctx->deslavement;
9801	struct mlxsw_sp_crif *crif;
9802	int err;
9803
9804	if (mlxsw_sp_dev_addr_list_empty(dev))
9805	return 0;
9806
9807	crif = mlxsw_sp_crif_lookup(router: ctx->mlxsw_sp->router, dev);
9808	if (!crif || crif->rif)
9809	return 0;
9810
9811	if (!mlxsw_sp_rif_should_config(rif: crif->rif, dev, event: NETDEV_UP))
9812	return 0;
9813
9814	err = __mlxsw_sp_inetaddr_event(mlxsw_sp: ctx->mlxsw_sp, dev, event: NETDEV_UP,
9815	nomaster, extack: ctx->extack);
9816	if (err)
9817	return err;
9818
9819	ctx->done++;
9820	return 0;
9821	}
9822
9823	static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev,
9824	struct netdev_nested_priv *priv)
9825	{
9826	struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data;
9827	bool nomaster = ctx->deslavement;
9828	struct mlxsw_sp_crif *crif;
9829
9830	if (!ctx->done)
9831	return 0;
9832
9833	if (mlxsw_sp_dev_addr_list_empty(dev))
9834	return 0;
9835
9836	crif = mlxsw_sp_crif_lookup(router: ctx->mlxsw_sp->router, dev);
9837	if (!crif || !crif->rif)
9838	return 0;
9839
9840	/* We are rolling back NETDEV_UP, so ask for that. */
9841	if (!mlxsw_sp_rif_should_config(rif: crif->rif, dev, event: NETDEV_UP))
9842	return 0;
9843
9844	__mlxsw_sp_inetaddr_event(mlxsw_sp: ctx->mlxsw_sp, dev, event: NETDEV_DOWN, nomaster,
9845	NULL);
9846
9847	ctx->done--;
9848	return 0;
9849	}
9850
9851	int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp,
9852	struct net_device *upper_dev,
9853	struct netlink_ext_ack *extack)
9854	{
9855	struct mlxsw_sp_router_replay_inetaddr_up ctx = {
9856	.mlxsw_sp = mlxsw_sp,
9857	.extack = extack,
9858	.deslavement = false,
9859	};
9860	struct netdev_nested_priv priv = {
9861	.data = &ctx,
9862	};
9863	int err;
9864
9865	err = mlxsw_sp_router_replay_inetaddr_up(dev: upper_dev, priv: &priv);
9866	if (err)
9867	return err;
9868
9869	err = netdev_walk_all_upper_dev_rcu(dev: upper_dev,
9870	fn: mlxsw_sp_router_replay_inetaddr_up,
9871	priv: &priv);
9872	if (err)
9873	goto err_replay_up;
9874
9875	return 0;
9876
9877	err_replay_up:
9878	netdev_walk_all_upper_dev_rcu(dev: upper_dev,
9879	fn: mlxsw_sp_router_unreplay_inetaddr_up,
9880	priv: &priv);
9881	mlxsw_sp_router_unreplay_inetaddr_up(dev: upper_dev, priv: &priv);
9882	return err;
9883	}
9884
9885	void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp,
9886	struct net_device *dev)
9887	{
9888	struct mlxsw_sp_router_replay_inetaddr_up ctx = {
9889	.mlxsw_sp = mlxsw_sp,
9890	.deslavement = true,
9891	};
9892	struct netdev_nested_priv priv = {
9893	.data = &ctx,
9894	};
9895
9896	mlxsw_sp_router_replay_inetaddr_up(dev, priv: &priv);
9897	}
9898
9899	static int
9900	mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port,
9901	u16 vid, struct net_device *dev,
9902	struct netlink_ext_ack *extack)
9903	{
9904	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
9905
9906	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port,
9907	vid);
9908	if (WARN_ON(!mlxsw_sp_port_vlan))
9909	return -EINVAL;
9910
9911	return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan,
9912	l3_dev: dev, extack);
9913	}
9914
9915	static void
9916	mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid,
9917	struct net_device *dev)
9918	{
9919	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
9920
9921	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port,
9922	vid);
9923	if (WARN_ON(!mlxsw_sp_port_vlan))
9924	return;
9925
9926	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
9927	}
9928
9929	static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9930	struct net_device *lag_dev,
9931	struct netlink_ext_ack *extack)
9932	{
9933	u16 default_vid = MLXSW_SP_DEFAULT_VID;
9934	struct net_device *upper_dev;
9935	struct list_head *iter;
9936	int done = 0;
9937	u16 vid;
9938	int err;
9939
9940	err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid: default_vid,
9941	dev: lag_dev, extack);
9942	if (err)
9943	return err;
9944
9945	netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9946	if (!is_vlan_dev(dev: upper_dev))
9947	continue;
9948
9949	vid = vlan_dev_vlan_id(dev: upper_dev);
9950	err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid,
9951	dev: upper_dev, extack);
9952	if (err)
9953	goto err_router_join_dev;
9954
9955	++done;
9956	}
9957
9958	return 0;
9959
9960	err_router_join_dev:
9961	netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9962	if (!is_vlan_dev(dev: upper_dev))
9963	continue;
9964	if (!done--)
9965	break;
9966
9967	vid = vlan_dev_vlan_id(dev: upper_dev);
9968	mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, dev: upper_dev);
9969	}
9970
9971	mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid: default_vid, dev: lag_dev);
9972	return err;
9973	}
9974
9975	static void
9976	__mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9977	struct net_device *lag_dev)
9978	{
9979	u16 default_vid = MLXSW_SP_DEFAULT_VID;
9980	struct net_device *upper_dev;
9981	struct list_head *iter;
9982	u16 vid;
9983
9984	netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9985	if (!is_vlan_dev(dev: upper_dev))
9986	continue;
9987
9988	vid = vlan_dev_vlan_id(dev: upper_dev);
9989	mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, dev: upper_dev);
9990	}
9991
9992	mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid: default_vid, dev: lag_dev);
9993	}
9994
9995	int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9996	struct net_device *lag_dev,
9997	struct netlink_ext_ack *extack)
9998	{
9999	int err;
10000
10001	mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10002	err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack);
10003	mutex_unlock(lock: &mlxsw_sp_port->mlxsw_sp->router->lock);
10004
10005	return err;
10006	}
10007
10008	void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port,
10009	struct net_device *lag_dev)
10010	{
10011	mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10012	__mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev);
10013	mutex_unlock(lock: &mlxsw_sp_port->mlxsw_sp->router->lock);
10014	}
10015
10016	static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb,
10017	unsigned long event, void *ptr)
10018	{
10019	struct net_device *dev = netdev_notifier_info_to_dev(info: ptr);
10020	struct mlxsw_sp_router *router;
10021	struct mlxsw_sp *mlxsw_sp;
10022	int err = 0;
10023
10024	router = container_of(nb, struct mlxsw_sp_router, netdevice_nb);
10025	mlxsw_sp = router->mlxsw_sp;
10026
10027	mutex_lock(&mlxsw_sp->router->lock);
10028
10029	if (event == NETDEV_REGISTER) {
10030	err = mlxsw_sp_netdevice_register(router, dev);
10031	if (err)
10032	/* No need to roll this back, UNREGISTER will collect it
10033	* anyhow.
10034	*/
10035	goto out;
10036	}
10037
10038	if (mlxsw_sp_is_offload_xstats_event(event))
10039	err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev,
10040	event, info: ptr);
10041	else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev))
10042	err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, ol_dev: dev,
10043	event, info: ptr);
10044	else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev))
10045	err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ul_dev: dev,
10046	event, info: ptr);
10047	else if (mlxsw_sp_is_router_event(event))
10048	err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr);
10049	else if (mlxsw_sp_is_vrf_event(event, ptr))
10050	err = mlxsw_sp_netdevice_vrf_event(l3_dev: dev, event, info: ptr);
10051
10052	if (event == NETDEV_UNREGISTER)
10053	mlxsw_sp_netdevice_unregister(router, dev);
10054
10055	out:
10056	mutex_unlock(lock: &mlxsw_sp->router->lock);
10057
10058	return notifier_from_errno(err);
10059	}
10060
10061	struct mlxsw_sp_macvlan_replay {
10062	struct mlxsw_sp *mlxsw_sp;
10063	struct netlink_ext_ack *extack;
10064	};
10065
10066	static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev,
10067	struct netdev_nested_priv *priv)
10068	{
10069	const struct mlxsw_sp_macvlan_replay *rms = priv->data;
10070	struct netlink_ext_ack *extack = rms->extack;
10071	struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp;
10072
10073	if (!netif_is_macvlan(dev))
10074	return 0;
10075
10076	return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev: dev, extack);
10077	}
10078
10079	static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif,
10080	struct netlink_ext_ack *extack)
10081	{
10082	struct mlxsw_sp_macvlan_replay rms = {
10083	.mlxsw_sp = rif->mlxsw_sp,
10084	.extack = extack,
10085	};
10086	struct netdev_nested_priv priv = {
10087	.data = &rms,
10088	};
10089
10090	return netdev_walk_all_upper_dev_rcu(dev: mlxsw_sp_rif_dev(rif),
10091	fn: mlxsw_sp_macvlan_replay_upper,
10092	priv: &priv);
10093	}
10094
10095	static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
10096	struct netdev_nested_priv *priv)
10097	{
10098	struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
10099
10100	if (!netif_is_macvlan(dev))
10101	return 0;
10102
10103	return mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10104	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
10105	}
10106
10107	static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
10108	{
10109	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10110	struct netdev_nested_priv priv = {
10111	.data = (void *)rif,
10112	};
10113
10114	if (!netif_is_macvlan_port(dev))
10115	return 0;
10116
10117	return netdev_walk_all_upper_dev_rcu(dev,
10118	fn: __mlxsw_sp_rif_macvlan_flush, priv: &priv);
10119	}
10120
10121	static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
10122	const struct mlxsw_sp_rif_params *params)
10123	{
10124	struct mlxsw_sp_rif_subport *rif_subport;
10125
10126	rif_subport = mlxsw_sp_rif_subport_rif(rif);
10127	refcount_set(r: &rif_subport->ref_count, n: 1);
10128	rif_subport->vid = params->vid;
10129	rif_subport->lag = params->lag;
10130	if (params->lag)
10131	rif_subport->lag_id = params->lag_id;
10132	else
10133	rif_subport->system_port = params->system_port;
10134	}
10135
10136	static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
10137	{
10138	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10139	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10140	struct mlxsw_sp_rif_subport *rif_subport;
10141	char ritr_pl[MLXSW_REG_RITR_LEN];
10142	u16 efid;
10143
10144	rif_subport = mlxsw_sp_rif_subport_rif(rif);
10145	mlxsw_reg_ritr_pack(payload: ritr_pl, enable, type: MLXSW_REG_RITR_SP_IF,
10146	rif: rif->rif_index, vr_id: rif->vr_id, mtu: dev->mtu);
10147	mlxsw_reg_ritr_mac_pack(payload: ritr_pl, mac: dev->dev_addr);
10148	mlxsw_reg_ritr_if_mac_profile_id_set(buf: ritr_pl, val: rif->mac_profile_id);
10149	efid = mlxsw_sp_fid_index(fid: rif->fid);
10150	mlxsw_reg_ritr_sp_if_pack(payload: ritr_pl, lag: rif_subport->lag,
10151	system_port: rif_subport->lag ? rif_subport->lag_id :
10152	rif_subport->system_port,
10153	efid, vid: 0);
10154	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
10155	}
10156
10157	static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif,
10158	struct netlink_ext_ack *extack)
10159	{
10160	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10161	u8 mac_profile;
10162	int err;
10163
10164	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp: rif->mlxsw_sp, mac: rif->addr,
10165	p_mac_profile: &mac_profile, extack);
10166	if (err)
10167	return err;
10168	rif->mac_profile_id = mac_profile;
10169
10170	err = mlxsw_sp_rif_subport_op(rif, enable: true);
10171	if (err)
10172	goto err_rif_subport_op;
10173
10174	err = mlxsw_sp_macvlan_replay(rif, extack);
10175	if (err)
10176	goto err_macvlan_replay;
10177
10178	err = mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10179	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: true);
10180	if (err)
10181	goto err_rif_fdb_op;
10182
10183	err = mlxsw_sp_fid_rif_set(fid: rif->fid, rif);
10184	if (err)
10185	goto err_fid_rif_set;
10186
10187	return 0;
10188
10189	err_fid_rif_set:
10190	mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10191	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
10192	err_rif_fdb_op:
10193	mlxsw_sp_rif_macvlan_flush(rif);
10194	err_macvlan_replay:
10195	mlxsw_sp_rif_subport_op(rif, enable: false);
10196	err_rif_subport_op:
10197	mlxsw_sp_rif_mac_profile_put(mlxsw_sp: rif->mlxsw_sp, mac_profile);
10198	return err;
10199	}
10200
10201	static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
10202	{
10203	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10204	struct mlxsw_sp_fid *fid = rif->fid;
10205
10206	mlxsw_sp_fid_rif_unset(fid);
10207	mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10208	fid: mlxsw_sp_fid_index(fid), adding: false);
10209	mlxsw_sp_rif_macvlan_flush(rif);
10210	mlxsw_sp_rif_subport_op(rif, enable: false);
10211	mlxsw_sp_rif_mac_profile_put(mlxsw_sp: rif->mlxsw_sp, mac_profile: rif->mac_profile_id);
10212	}
10213
10214	static struct mlxsw_sp_fid *
10215	mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
10216	const struct mlxsw_sp_rif_params *params,
10217	struct netlink_ext_ack *extack)
10218	{
10219	return mlxsw_sp_fid_rfid_get(mlxsw_sp: rif->mlxsw_sp, rif_index: rif->rif_index);
10220	}
10221
10222	static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
10223	.type = MLXSW_SP_RIF_TYPE_SUBPORT,
10224	.rif_size = sizeof(struct mlxsw_sp_rif_subport),
10225	.setup = mlxsw_sp_rif_subport_setup,
10226	.configure = mlxsw_sp_rif_subport_configure,
10227	.deconfigure = mlxsw_sp_rif_subport_deconfigure,
10228	.fid_get = mlxsw_sp_rif_subport_fid_get,
10229	};
10230
10231	static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable)
10232	{
10233	enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF;
10234	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10235	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10236	char ritr_pl[MLXSW_REG_RITR_LEN];
10237
10238	mlxsw_reg_ritr_pack(payload: ritr_pl, enable, type, rif: rif->rif_index, vr_id: rif->vr_id,
10239	mtu: dev->mtu);
10240	mlxsw_reg_ritr_mac_pack(payload: ritr_pl, mac: dev->dev_addr);
10241	mlxsw_reg_ritr_if_mac_profile_id_set(buf: ritr_pl, val: rif->mac_profile_id);
10242	mlxsw_reg_ritr_fid_if_fid_set(buf: ritr_pl, val: fid);
10243
10244	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
10245	}
10246
10247	u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
10248	{
10249	return mlxsw_core_max_ports(mlxsw_core: mlxsw_sp->core) + 1;
10250	}
10251
10252	static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif,
10253	struct netlink_ext_ack *extack)
10254	{
10255	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10256	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10257	u16 fid_index = mlxsw_sp_fid_index(fid: rif->fid);
10258	u8 mac_profile;
10259	int err;
10260
10261	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, mac: rif->addr,
10262	p_mac_profile: &mac_profile, extack);
10263	if (err)
10264	return err;
10265	rif->mac_profile_id = mac_profile;
10266
10267	err = mlxsw_sp_rif_fid_op(rif, fid: fid_index, enable: true);
10268	if (err)
10269	goto err_rif_fid_op;
10270
10271	err = mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_MC,
10272	local_port: mlxsw_sp_router_port(mlxsw_sp), member: true);
10273	if (err)
10274	goto err_fid_mc_flood_set;
10275
10276	err = mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_BC,
10277	local_port: mlxsw_sp_router_port(mlxsw_sp), member: true);
10278	if (err)
10279	goto err_fid_bc_flood_set;
10280
10281	err = mlxsw_sp_macvlan_replay(rif, extack);
10282	if (err)
10283	goto err_macvlan_replay;
10284
10285	err = mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10286	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: true);
10287	if (err)
10288	goto err_rif_fdb_op;
10289
10290	err = mlxsw_sp_fid_rif_set(fid: rif->fid, rif);
10291	if (err)
10292	goto err_fid_rif_set;
10293
10294	return 0;
10295
10296	err_fid_rif_set:
10297	mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10298	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
10299	err_rif_fdb_op:
10300	mlxsw_sp_rif_macvlan_flush(rif);
10301	err_macvlan_replay:
10302	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_BC,
10303	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10304	err_fid_bc_flood_set:
10305	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_MC,
10306	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10307	err_fid_mc_flood_set:
10308	mlxsw_sp_rif_fid_op(rif, fid: fid_index, enable: false);
10309	err_rif_fid_op:
10310	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
10311	return err;
10312	}
10313
10314	static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
10315	{
10316	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10317	u16 fid_index = mlxsw_sp_fid_index(fid: rif->fid);
10318	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10319	struct mlxsw_sp_fid *fid = rif->fid;
10320
10321	mlxsw_sp_fid_rif_unset(fid);
10322	mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10323	fid: mlxsw_sp_fid_index(fid), adding: false);
10324	mlxsw_sp_rif_macvlan_flush(rif);
10325	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_BC,
10326	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10327	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_MC,
10328	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10329	mlxsw_sp_rif_fid_op(rif, fid: fid_index, enable: false);
10330	mlxsw_sp_rif_mac_profile_put(mlxsw_sp: rif->mlxsw_sp, mac_profile: rif->mac_profile_id);
10331	}
10332
10333	static struct mlxsw_sp_fid *
10334	mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
10335	const struct mlxsw_sp_rif_params *params,
10336	struct netlink_ext_ack *extack)
10337	{
10338	int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif);
10339
10340	return mlxsw_sp_fid_8021d_get(mlxsw_sp: rif->mlxsw_sp, br_ifindex: rif_ifindex);
10341	}
10342
10343	static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
10344	{
10345	struct switchdev_notifier_fdb_info info = {};
10346	struct net_device *dev;
10347
10348	dev = br_fdb_find_port(br_dev: mlxsw_sp_rif_dev(rif), addr: mac, vid: 0);
10349	if (!dev)
10350	return;
10351
10352	info.addr = mac;
10353	info.vid = 0;
10354	call_switchdev_notifiers(val: SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, info: &info.info,
10355	NULL);
10356	}
10357
10358	static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
10359	.type = MLXSW_SP_RIF_TYPE_FID,
10360	.rif_size = sizeof(struct mlxsw_sp_rif),
10361	.configure = mlxsw_sp_rif_fid_configure,
10362	.deconfigure = mlxsw_sp_rif_fid_deconfigure,
10363	.fid_get = mlxsw_sp_rif_fid_fid_get,
10364	.fdb_del = mlxsw_sp_rif_fid_fdb_del,
10365	};
10366
10367	static struct mlxsw_sp_fid *
10368	mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
10369	const struct mlxsw_sp_rif_params *params,
10370	struct netlink_ext_ack *extack)
10371	{
10372	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10373	struct net_device *br_dev;
10374
10375	if (WARN_ON(!params->vid))
10376	return ERR_PTR(error: -EINVAL);
10377
10378	if (is_vlan_dev(dev)) {
10379	br_dev = vlan_dev_real_dev(dev);
10380	if (WARN_ON(!netif_is_bridge_master(br_dev)))
10381	return ERR_PTR(error: -EINVAL);
10382	}
10383
10384	return mlxsw_sp_fid_8021q_get(mlxsw_sp: rif->mlxsw_sp, vid: params->vid);
10385	}
10386
10387	static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
10388	{
10389	struct net_device *rif_dev = mlxsw_sp_rif_dev(rif);
10390	struct switchdev_notifier_fdb_info info = {};
10391	u16 vid = mlxsw_sp_fid_8021q_vid(fid: rif->fid);
10392	struct net_device *br_dev;
10393	struct net_device *dev;
10394
10395	br_dev = is_vlan_dev(dev: rif_dev) ? vlan_dev_real_dev(dev: rif_dev) : rif_dev;
10396	dev = br_fdb_find_port(br_dev, addr: mac, vid);
10397	if (!dev)
10398	return;
10399
10400	info.addr = mac;
10401	info.vid = vid;
10402	call_switchdev_notifiers(val: SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, info: &info.info,
10403	NULL);
10404	}
10405
10406	static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid,
10407	bool enable)
10408	{
10409	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10410	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10411	char ritr_pl[MLXSW_REG_RITR_LEN];
10412
10413	mlxsw_reg_ritr_vlan_if_pack(payload: ritr_pl, enable, rif: rif->rif_index, vr_id: rif->vr_id,
10414	mtu: dev->mtu, mac: dev->dev_addr,
10415	mac_profile_id: rif->mac_profile_id, vlan_id: vid, efid);
10416
10417	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
10418	}
10419
10420	static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid,
10421	struct netlink_ext_ack *extack)
10422	{
10423	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10424	u16 vid = mlxsw_sp_fid_8021q_vid(fid: rif->fid);
10425	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10426	u8 mac_profile;
10427	int err;
10428
10429	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, mac: rif->addr,
10430	p_mac_profile: &mac_profile, extack);
10431	if (err)
10432	return err;
10433	rif->mac_profile_id = mac_profile;
10434
10435	err = mlxsw_sp_rif_vlan_op(rif, vid, efid, enable: true);
10436	if (err)
10437	goto err_rif_vlan_fid_op;
10438
10439	err = mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_MC,
10440	local_port: mlxsw_sp_router_port(mlxsw_sp), member: true);
10441	if (err)
10442	goto err_fid_mc_flood_set;
10443
10444	err = mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_BC,
10445	local_port: mlxsw_sp_router_port(mlxsw_sp), member: true);
10446	if (err)
10447	goto err_fid_bc_flood_set;
10448
10449	err = mlxsw_sp_macvlan_replay(rif, extack);
10450	if (err)
10451	goto err_macvlan_replay;
10452
10453	err = mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10454	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: true);
10455	if (err)
10456	goto err_rif_fdb_op;
10457
10458	err = mlxsw_sp_fid_rif_set(fid: rif->fid, rif);
10459	if (err)
10460	goto err_fid_rif_set;
10461
10462	return 0;
10463
10464	err_fid_rif_set:
10465	mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10466	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
10467	err_rif_fdb_op:
10468	mlxsw_sp_rif_macvlan_flush(rif);
10469	err_macvlan_replay:
10470	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_BC,
10471	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10472	err_fid_bc_flood_set:
10473	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_MC,
10474	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10475	err_fid_mc_flood_set:
10476	mlxsw_sp_rif_vlan_op(rif, vid, efid: 0, enable: false);
10477	err_rif_vlan_fid_op:
10478	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
10479	return err;
10480	}
10481
10482	static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
10483	{
10484	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10485	u16 vid = mlxsw_sp_fid_8021q_vid(fid: rif->fid);
10486	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10487
10488	mlxsw_sp_fid_rif_unset(fid: rif->fid);
10489	mlxsw_sp_rif_fdb_op(mlxsw_sp: rif->mlxsw_sp, mac: dev->dev_addr,
10490	fid: mlxsw_sp_fid_index(fid: rif->fid), adding: false);
10491	mlxsw_sp_rif_macvlan_flush(rif);
10492	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_BC,
10493	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10494	mlxsw_sp_fid_flood_set(fid: rif->fid, packet_type: MLXSW_SP_FLOOD_TYPE_MC,
10495	local_port: mlxsw_sp_router_port(mlxsw_sp), member: false);
10496	mlxsw_sp_rif_vlan_op(rif, vid, efid: 0, enable: false);
10497	mlxsw_sp_rif_mac_profile_put(mlxsw_sp: rif->mlxsw_sp, mac_profile: rif->mac_profile_id);
10498	}
10499
10500	static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif,
10501	struct netlink_ext_ack *extack)
10502	{
10503	return mlxsw_sp_rif_vlan_configure(rif, efid: 0, extack);
10504	}
10505
10506	static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = {
10507	.type = MLXSW_SP_RIF_TYPE_VLAN,
10508	.rif_size = sizeof(struct mlxsw_sp_rif),
10509	.configure = mlxsw_sp1_rif_vlan_configure,
10510	.deconfigure = mlxsw_sp_rif_vlan_deconfigure,
10511	.fid_get = mlxsw_sp_rif_vlan_fid_get,
10512	.fdb_del = mlxsw_sp_rif_vlan_fdb_del,
10513	};
10514
10515	static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif,
10516	struct netlink_ext_ack *extack)
10517	{
10518	u16 efid = mlxsw_sp_fid_index(fid: rif->fid);
10519
10520	return mlxsw_sp_rif_vlan_configure(rif, efid, extack);
10521	}
10522
10523	static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = {
10524	.type = MLXSW_SP_RIF_TYPE_VLAN,
10525	.rif_size = sizeof(struct mlxsw_sp_rif),
10526	.configure = mlxsw_sp2_rif_vlan_configure,
10527	.deconfigure = mlxsw_sp_rif_vlan_deconfigure,
10528	.fid_get = mlxsw_sp_rif_vlan_fid_get,
10529	.fdb_del = mlxsw_sp_rif_vlan_fdb_del,
10530	};
10531
10532	static struct mlxsw_sp_rif_ipip_lb *
10533	mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
10534	{
10535	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
10536	}
10537
10538	static void
10539	mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
10540	const struct mlxsw_sp_rif_params *params)
10541	{
10542	struct mlxsw_sp_rif_params_ipip_lb *params_lb;
10543	struct mlxsw_sp_rif_ipip_lb *rif_lb;
10544
10545	params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
10546	common);
10547	rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
10548	rif_lb->lb_config = params_lb->lb_config;
10549	}
10550
10551	static int
10552	mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
10553	struct netlink_ext_ack *extack)
10554	{
10555	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10556	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10557	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev: dev);
10558	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10559	struct mlxsw_sp_vr *ul_vr;
10560	int err;
10561
10562	ul_vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id: ul_tb_id, extack);
10563	if (IS_ERR(ptr: ul_vr))
10564	return PTR_ERR(ptr: ul_vr);
10565
10566	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr_id: ul_vr->id, ul_rif_id: 0, enable: true);
10567	if (err)
10568	goto err_loopback_op;
10569
10570	lb_rif->ul_vr_id = ul_vr->id;
10571	lb_rif->ul_rif_id = 0;
10572	++ul_vr->rif_count;
10573	return 0;
10574
10575	err_loopback_op:
10576	mlxsw_sp_vr_put(mlxsw_sp, vr: ul_vr);
10577	return err;
10578	}
10579
10580	static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
10581	{
10582	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10583	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10584	struct mlxsw_sp_vr *ul_vr;
10585
10586	ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
10587	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr_id: ul_vr->id, ul_rif_id: 0, enable: false);
10588
10589	--ul_vr->rif_count;
10590	mlxsw_sp_vr_put(mlxsw_sp, vr: ul_vr);
10591	}
10592
10593	static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
10594	.type = MLXSW_SP_RIF_TYPE_IPIP_LB,
10595	.rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
10596	.setup = mlxsw_sp_rif_ipip_lb_setup,
10597	.configure = mlxsw_sp1_rif_ipip_lb_configure,
10598	.deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure,
10599	};
10600
10601	static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
10602	[MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
10603	[MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp1_rif_vlan_ops,
10604	[MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
10605	[MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops,
10606	};
10607
10608	static int
10609	mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
10610	{
10611	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10612	char ritr_pl[MLXSW_REG_RITR_LEN];
10613
10614	mlxsw_reg_ritr_pack(payload: ritr_pl, enable, type: MLXSW_REG_RITR_LOOPBACK_IF,
10615	rif: ul_rif->rif_index, vr_id: ul_rif->vr_id, IP_MAX_MTU);
10616	mlxsw_reg_ritr_loopback_protocol_set(buf: ritr_pl,
10617	val: MLXSW_REG_RITR_LOOPBACK_GENERIC);
10618
10619	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(ritr), payload: ritr_pl);
10620	}
10621
10622	static struct mlxsw_sp_rif *
10623	mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
10624	struct mlxsw_sp_crif *ul_crif,
10625	struct netlink_ext_ack *extack)
10626	{
10627	struct mlxsw_sp_rif *ul_rif;
10628	u8 rif_entries = 1;
10629	u16 rif_index;
10630	int err;
10631
10632	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, p_rif_index: &rif_index, rif_entries);
10633	if (err) {
10634	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
10635	return ERR_PTR(error: err);
10636	}
10637
10638	ul_rif = mlxsw_sp_rif_alloc(rif_size: sizeof(*ul_rif), rif_index, vr_id: vr->id,
10639	crif: ul_crif);
10640	if (!ul_rif) {
10641	err = -ENOMEM;
10642	goto err_rif_alloc;
10643	}
10644
10645	mlxsw_sp->router->rifs[rif_index] = ul_rif;
10646	ul_rif->mlxsw_sp = mlxsw_sp;
10647	ul_rif->rif_entries = rif_entries;
10648	err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, enable: true);
10649	if (err)
10650	goto ul_rif_op_err;
10651
10652	atomic_add(i: rif_entries, v: &mlxsw_sp->router->rifs_count);
10653	return ul_rif;
10654
10655	ul_rif_op_err:
10656	mlxsw_sp->router->rifs[rif_index] = NULL;
10657	mlxsw_sp_rif_free(rif: ul_rif);
10658	err_rif_alloc:
10659	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
10660	return ERR_PTR(error: err);
10661	}
10662
10663	static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
10664	{
10665	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10666	u8 rif_entries = ul_rif->rif_entries;
10667	u16 rif_index = ul_rif->rif_index;
10668
10669	atomic_sub(i: rif_entries, v: &mlxsw_sp->router->rifs_count);
10670	mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, enable: false);
10671	mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
10672	mlxsw_sp_rif_free(rif: ul_rif);
10673	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
10674	}
10675
10676	static struct mlxsw_sp_rif *
10677	mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
10678	struct mlxsw_sp_crif *ul_crif,
10679	struct netlink_ext_ack *extack)
10680	{
10681	struct mlxsw_sp_vr *vr;
10682	int err;
10683
10684	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
10685	if (IS_ERR(ptr: vr))
10686	return ERR_CAST(ptr: vr);
10687
10688	if (refcount_inc_not_zero(r: &vr->ul_rif_refcnt))
10689	return vr->ul_rif;
10690
10691	vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack);
10692	if (IS_ERR(ptr: vr->ul_rif)) {
10693	err = PTR_ERR(ptr: vr->ul_rif);
10694	goto err_ul_rif_create;
10695	}
10696
10697	vr->rif_count++;
10698	refcount_set(r: &vr->ul_rif_refcnt, n: 1);
10699
10700	return vr->ul_rif;
10701
10702	err_ul_rif_create:
10703	mlxsw_sp_vr_put(mlxsw_sp, vr);
10704	return ERR_PTR(error: err);
10705	}
10706
10707	static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
10708	{
10709	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10710	struct mlxsw_sp_vr *vr;
10711
10712	vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
10713
10714	if (!refcount_dec_and_test(r: &vr->ul_rif_refcnt))
10715	return;
10716
10717	vr->rif_count--;
10718	mlxsw_sp_ul_rif_destroy(ul_rif);
10719	mlxsw_sp_vr_put(mlxsw_sp, vr);
10720	}
10721
10722	int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
10723	u16 *ul_rif_index)
10724	{
10725	struct mlxsw_sp_rif *ul_rif;
10726	int err = 0;
10727
10728	mutex_lock(&mlxsw_sp->router->lock);
10729	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, tb_id: ul_tb_id, NULL, NULL);
10730	if (IS_ERR(ptr: ul_rif)) {
10731	err = PTR_ERR(ptr: ul_rif);
10732	goto out;
10733	}
10734	*ul_rif_index = ul_rif->rif_index;
10735	out:
10736	mutex_unlock(lock: &mlxsw_sp->router->lock);
10737	return err;
10738	}
10739
10740	void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
10741	{
10742	struct mlxsw_sp_rif *ul_rif;
10743
10744	mutex_lock(&mlxsw_sp->router->lock);
10745	ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
10746	if (WARN_ON(!ul_rif))
10747	goto out;
10748
10749	mlxsw_sp_ul_rif_put(ul_rif);
10750	out:
10751	mutex_unlock(lock: &mlxsw_sp->router->lock);
10752	}
10753
10754	static int
10755	mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
10756	struct netlink_ext_ack *extack)
10757	{
10758	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10759	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10760	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev: dev);
10761	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10762	struct mlxsw_sp_rif *ul_rif;
10763	int err;
10764
10765	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, tb_id: ul_tb_id, NULL, extack);
10766	if (IS_ERR(ptr: ul_rif))
10767	return PTR_ERR(ptr: ul_rif);
10768
10769	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr_id: 0, ul_rif_id: ul_rif->rif_index, enable: true);
10770	if (err)
10771	goto err_loopback_op;
10772
10773	lb_rif->ul_vr_id = 0;
10774	lb_rif->ul_rif_id = ul_rif->rif_index;
10775
10776	return 0;
10777
10778	err_loopback_op:
10779	mlxsw_sp_ul_rif_put(ul_rif);
10780	return err;
10781	}
10782
10783	static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
10784	{
10785	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10786	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10787	struct mlxsw_sp_rif *ul_rif;
10788
10789	ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, rif_index: lb_rif->ul_rif_id);
10790	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr_id: 0, ul_rif_id: lb_rif->ul_rif_id, enable: false);
10791	mlxsw_sp_ul_rif_put(ul_rif);
10792	}
10793
10794	static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
10795	.type = MLXSW_SP_RIF_TYPE_IPIP_LB,
10796	.rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
10797	.setup = mlxsw_sp_rif_ipip_lb_setup,
10798	.configure = mlxsw_sp2_rif_ipip_lb_configure,
10799	.deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure,
10800	};
10801
10802	static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
10803	[MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
10804	[MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp2_rif_vlan_ops,
10805	[MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
10806	[MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops,
10807	};
10808
10809	static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp)
10810	{
10811	struct gen_pool *rifs_table;
10812	int err;
10813
10814	rifs_table = gen_pool_create(0, -1);
10815	if (!rifs_table)
10816	return -ENOMEM;
10817
10818	gen_pool_set_algo(pool: rifs_table, algo: gen_pool_first_fit_order_align,
10819	NULL);
10820
10821	err = gen_pool_add(pool: rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET,
10822	MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), nid: -1);
10823	if (err)
10824	goto err_gen_pool_add;
10825
10826	mlxsw_sp->router->rifs_table = rifs_table;
10827
10828	return 0;
10829
10830	err_gen_pool_add:
10831	gen_pool_destroy(rifs_table);
10832	return err;
10833	}
10834
10835	static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp)
10836	{
10837	gen_pool_destroy(mlxsw_sp->router->rifs_table);
10838	}
10839
10840	static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
10841	{
10842	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10843	struct devlink *devlink = priv_to_devlink(priv: mlxsw_sp->core);
10844	struct mlxsw_core *core = mlxsw_sp->core;
10845	int err;
10846
10847	if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES))
10848	return -EIO;
10849	mlxsw_sp->router->max_rif_mac_profile =
10850	MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES);
10851
10852	mlxsw_sp->router->rifs = kcalloc(n: max_rifs,
10853	size: sizeof(struct mlxsw_sp_rif *),
10854	GFP_KERNEL);
10855	if (!mlxsw_sp->router->rifs)
10856	return -ENOMEM;
10857
10858	err = mlxsw_sp_rifs_table_init(mlxsw_sp);
10859	if (err)
10860	goto err_rifs_table_init;
10861
10862	idr_init(idr: &mlxsw_sp->router->rif_mac_profiles_idr);
10863	atomic_set(v: &mlxsw_sp->router->rif_mac_profiles_count, i: 0);
10864	atomic_set(v: &mlxsw_sp->router->rifs_count, i: 0);
10865	devl_resource_occ_get_register(devlink,
10866	resource_id: MLXSW_SP_RESOURCE_RIF_MAC_PROFILES,
10867	occ_get: mlxsw_sp_rif_mac_profiles_occ_get,
10868	occ_get_priv: mlxsw_sp);
10869	devl_resource_occ_get_register(devlink,
10870	resource_id: MLXSW_SP_RESOURCE_RIFS,
10871	occ_get: mlxsw_sp_rifs_occ_get,
10872	occ_get_priv: mlxsw_sp);
10873
10874	return 0;
10875
10876	err_rifs_table_init:
10877	kfree(objp: mlxsw_sp->router->rifs);
10878	return err;
10879	}
10880
10881	static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
10882	{
10883	int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10884	struct devlink *devlink = priv_to_devlink(priv: mlxsw_sp->core);
10885	int i;
10886
10887	WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count));
10888	for (i = 0; i < max_rifs; i++)
10889	WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
10890
10891	devl_resource_occ_get_unregister(devlink, resource_id: MLXSW_SP_RESOURCE_RIFS);
10892	devl_resource_occ_get_unregister(devlink,
10893	resource_id: MLXSW_SP_RESOURCE_RIF_MAC_PROFILES);
10894	WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr));
10895	idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr);
10896	mlxsw_sp_rifs_table_fini(mlxsw_sp);
10897	kfree(objp: mlxsw_sp->router->rifs);
10898	}
10899
10900	static int
10901	mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
10902	{
10903	char tigcr_pl[MLXSW_REG_TIGCR_LEN];
10904
10905	mlxsw_reg_tigcr_pack(payload: tigcr_pl, ttlc: true, ttl_uc: 0);
10906	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(tigcr), payload: tigcr_pl);
10907	}
10908
10909	static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
10910	{
10911	int err;
10912
10913	INIT_LIST_HEAD(list: &mlxsw_sp->router->ipip_list);
10914
10915	err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
10916	if (err)
10917	return err;
10918	err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
10919	if (err)
10920	return err;
10921
10922	return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
10923	}
10924
10925	static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp)
10926	{
10927	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr;
10928	return mlxsw_sp_ipips_init(mlxsw_sp);
10929	}
10930
10931	static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp)
10932	{
10933	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr;
10934	return mlxsw_sp_ipips_init(mlxsw_sp);
10935	}
10936
10937	static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
10938	{
10939	WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
10940	}
10941
10942	static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
10943	{
10944	struct mlxsw_sp_router *router;
10945
10946	/* Flush pending FIB notifications and then flush the device's
10947	* table before requesting another dump. The FIB notification
10948	* block is unregistered, so no need to take RTNL.
10949	*/
10950	mlxsw_core_flush_owq();
10951	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
10952	mlxsw_sp_router_fib_flush(mlxsw_sp: router->mlxsw_sp);
10953	}
10954
10955	#ifdef CONFIG_IP_ROUTE_MULTIPATH
10956	struct mlxsw_sp_mp_hash_config {
10957	DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT);
10958	DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT);
10959	DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT);
10960	DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT);
10961	bool inc_parsing_depth;
10962	};
10963
10964	#define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \
10965	bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1)
10966
10967	#define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \
10968	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1)
10969
10970	#define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \
10971	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr)
10972
10973	static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config)
10974	{
10975	unsigned long *inner_headers = config->inner_headers;
10976	unsigned long *inner_fields = config->inner_fields;
10977
10978	/* IPv4 inner */
10979	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
10980	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
10981	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
10982	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
10983	/* IPv6 inner */
10984	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
10985	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
10986	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
10987	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
10988	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
10989	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
10990	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
10991	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
10992	}
10993
10994	static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
10995	{
10996	unsigned long *headers = config->headers;
10997	unsigned long *fields = config->fields;
10998
10999	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
11000	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
11001	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
11002	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
11003	}
11004
11005	static void
11006	mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config,
11007	u32 hash_fields)
11008	{
11009	unsigned long *inner_headers = config->inner_headers;
11010	unsigned long *inner_fields = config->inner_fields;
11011
11012	/* IPv4 Inner */
11013	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
11014	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
11015	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
11016	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
11017	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
11018	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
11019	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
11020	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL);
11021	/* IPv6 inner */
11022	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
11023	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
11024	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) {
11025	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
11026	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
11027	}
11028	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) {
11029	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
11030	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
11031	}
11032	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
11033	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
11034	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
11035	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
11036	/* L4 inner */
11037	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4);
11038	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6);
11039	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
11040	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT);
11041	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
11042	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT);
11043	}
11044
11045	static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp,
11046	struct mlxsw_sp_mp_hash_config *config)
11047	{
11048	struct net *net = mlxsw_sp_net(mlxsw_sp);
11049	unsigned long *headers = config->headers;
11050	unsigned long *fields = config->fields;
11051	u32 hash_fields;
11052
11053	switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) {
11054	case 0:
11055	mlxsw_sp_mp4_hash_outer_addr(config);
11056	break;
11057	case 1:
11058	mlxsw_sp_mp4_hash_outer_addr(config);
11059	MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
11060	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
11061	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11062	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11063	break;
11064	case 2:
11065	/* Outer */
11066	mlxsw_sp_mp4_hash_outer_addr(config);
11067	/* Inner */
11068	mlxsw_sp_mp_hash_inner_l3(config);
11069	break;
11070	case 3:
11071	hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields);
11072	/* Outer */
11073	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
11074	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
11075	MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
11076	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
11077	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
11078	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
11079	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
11080	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
11081	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
11082	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
11083	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11084	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
11085	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11086	/* Inner */
11087	mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
11088	break;
11089	}
11090	}
11091
11092	static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
11093	{
11094	unsigned long *headers = config->headers;
11095	unsigned long *fields = config->fields;
11096
11097	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
11098	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
11099	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
11100	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
11101	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
11102	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
11103	}
11104
11105	static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp,
11106	struct mlxsw_sp_mp_hash_config *config)
11107	{
11108	u32 hash_fields = ip6_multipath_hash_fields(net: mlxsw_sp_net(mlxsw_sp));
11109	unsigned long *headers = config->headers;
11110	unsigned long *fields = config->fields;
11111
11112	switch (ip6_multipath_hash_policy(net: mlxsw_sp_net(mlxsw_sp))) {
11113	case 0:
11114	mlxsw_sp_mp6_hash_outer_addr(config);
11115	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11116	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11117	break;
11118	case 1:
11119	mlxsw_sp_mp6_hash_outer_addr(config);
11120	MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
11121	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11122	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11123	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11124	break;
11125	case 2:
11126	/* Outer */
11127	mlxsw_sp_mp6_hash_outer_addr(config);
11128	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11129	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11130	/* Inner */
11131	mlxsw_sp_mp_hash_inner_l3(config);
11132	config->inc_parsing_depth = true;
11133	break;
11134	case 3:
11135	/* Outer */
11136	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
11137	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
11138	MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
11139	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) {
11140	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
11141	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
11142	}
11143	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) {
11144	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
11145	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
11146	}
11147	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
11148	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11149	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
11150	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11151	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
11152	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11153	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
11154	MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11155	/* Inner */
11156	mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
11157	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)
11158	config->inc_parsing_depth = true;
11159	break;
11160	}
11161	}
11162
11163	static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp,
11164	bool old_inc_parsing_depth,
11165	bool new_inc_parsing_depth)
11166	{
11167	int err;
11168
11169	if (!old_inc_parsing_depth && new_inc_parsing_depth) {
11170	err = mlxsw_sp_parsing_depth_inc(mlxsw_sp);
11171	if (err)
11172	return err;
11173	mlxsw_sp->router->inc_parsing_depth = true;
11174	} else if (old_inc_parsing_depth && !new_inc_parsing_depth) {
11175	mlxsw_sp_parsing_depth_dec(mlxsw_sp);
11176	mlxsw_sp->router->inc_parsing_depth = false;
11177	}
11178
11179	return 0;
11180	}
11181
11182	static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
11183	{
11184	bool old_inc_parsing_depth, new_inc_parsing_depth;
11185	struct mlxsw_sp_mp_hash_config config = {};
11186	char recr2_pl[MLXSW_REG_RECR2_LEN];
11187	unsigned long bit;
11188	u32 seed;
11189	int err;
11190
11191	seed = jhash(key: mlxsw_sp->base_mac, length: sizeof(mlxsw_sp->base_mac), initval: 0);
11192	mlxsw_reg_recr2_pack(payload: recr2_pl, seed);
11193	mlxsw_sp_mp4_hash_init(mlxsw_sp, config: &config);
11194	mlxsw_sp_mp6_hash_init(mlxsw_sp, config: &config);
11195
11196	old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
11197	new_inc_parsing_depth = config.inc_parsing_depth;
11198	err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp,
11199	old_inc_parsing_depth,
11200	new_inc_parsing_depth);
11201	if (err)
11202	return err;
11203
11204	for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT)
11205	mlxsw_reg_recr2_outer_header_enables_set(buf: recr2_pl, index: bit, val: 1);
11206	for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT)
11207	mlxsw_reg_recr2_outer_header_fields_enable_set(buf: recr2_pl, index: bit, val: 1);
11208	for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT)
11209	mlxsw_reg_recr2_inner_header_enables_set(buf: recr2_pl, index: bit, val: 1);
11210	for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT)
11211	mlxsw_reg_recr2_inner_header_fields_enable_set(buf: recr2_pl, index: bit, val: 1);
11212
11213	err = mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(recr2), payload: recr2_pl);
11214	if (err)
11215	goto err_reg_write;
11216
11217	return 0;
11218
11219	err_reg_write:
11220	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth: new_inc_parsing_depth,
11221	new_inc_parsing_depth: old_inc_parsing_depth);
11222	return err;
11223	}
11224
11225	static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
11226	{
11227	bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
11228
11229	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth,
11230	new_inc_parsing_depth: false);
11231	}
11232	#else
11233	static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
11234	{
11235	return 0;
11236	}
11237
11238	static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
11239	{
11240	}
11241	#endif
11242
11243	static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
11244	{
11245	char rdpm_pl[MLXSW_REG_RDPM_LEN];
11246	unsigned int i;
11247
11248	MLXSW_REG_ZERO(rdpm, rdpm_pl);
11249
11250	/* HW is determining switch priority based on DSCP-bits, but the
11251	* kernel is still doing that based on the ToS. Since there's a
11252	* mismatch in bits we need to make sure to translate the right
11253	* value ToS would observe, skipping the 2 least-significant ECN bits.
11254	*/
11255	for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
11256	mlxsw_reg_rdpm_pack(payload: rdpm_pl, index: i, prio: rt_tos2priority(tos: i << 2));
11257
11258	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(rdpm), payload: rdpm_pl);
11259	}
11260
11261	static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
11262	{
11263	struct net *net = mlxsw_sp_net(mlxsw_sp);
11264	char rgcr_pl[MLXSW_REG_RGCR_LEN];
11265	u64 max_rifs;
11266	bool usp;
11267
11268	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
11269	return -EIO;
11270	max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
11271	usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority);
11272
11273	mlxsw_reg_rgcr_pack(payload: rgcr_pl, ipv4_en: true, ipv6_en: true);
11274	mlxsw_reg_rgcr_max_router_interfaces_set(buf: rgcr_pl, val: max_rifs);
11275	mlxsw_reg_rgcr_usp_set(buf: rgcr_pl, val: usp);
11276	return mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(rgcr), payload: rgcr_pl);
11277	}
11278
11279	static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
11280	{
11281	char rgcr_pl[MLXSW_REG_RGCR_LEN];
11282
11283	mlxsw_reg_rgcr_pack(payload: rgcr_pl, ipv4_en: false, ipv6_en: false);
11284	mlxsw_reg_write(mlxsw_core: mlxsw_sp->core, MLXSW_REG(rgcr), payload: rgcr_pl);
11285	}
11286
11287	static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp,
11288	struct netlink_ext_ack *extack)
11289	{
11290	struct mlxsw_sp_router *router = mlxsw_sp->router;
11291	struct mlxsw_sp_rif *lb_rif;
11292	int err;
11293
11294	router->lb_crif = mlxsw_sp_crif_alloc(NULL);
11295	if (!router->lb_crif)
11296	return -ENOMEM;
11297
11298	/* Create a generic loopback RIF associated with the main table
11299	* (default VRF). Any table can be used, but the main table exists
11300	* anyway, so we do not waste resources. Loopback RIFs are usually
11301	* created with a NULL CRIF, but this RIF is used as a fallback RIF
11302	* for blackhole nexthops, and nexthops expect to have a valid CRIF.
11303	*/
11304	lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, tb_id: RT_TABLE_MAIN, ul_crif: router->lb_crif,
11305	extack);
11306	if (IS_ERR(ptr: lb_rif)) {
11307	err = PTR_ERR(ptr: lb_rif);
11308	goto err_ul_rif_get;
11309	}
11310
11311	return 0;
11312
11313	err_ul_rif_get:
11314	mlxsw_sp_crif_free(crif: router->lb_crif);
11315	return err;
11316	}
11317
11318	static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp)
11319	{
11320	mlxsw_sp_ul_rif_put(ul_rif: mlxsw_sp->router->lb_crif->rif);
11321	mlxsw_sp_crif_free(crif: mlxsw_sp->router->lb_crif);
11322	}
11323
11324	static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp)
11325	{
11326	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges);
11327
11328	mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr;
11329	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges;
11330	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
11331
11332	return 0;
11333	}
11334
11335	const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = {
11336	.init = mlxsw_sp1_router_init,
11337	.ipips_init = mlxsw_sp1_ipips_init,
11338	};
11339
11340	static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp)
11341	{
11342	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges);
11343
11344	mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr;
11345	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges;
11346	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
11347
11348	return 0;
11349	}
11350
11351	const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = {
11352	.init = mlxsw_sp2_router_init,
11353	.ipips_init = mlxsw_sp2_ipips_init,
11354	};
11355
11356	int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
11357	struct netlink_ext_ack *extack)
11358	{
11359	struct mlxsw_sp_router *router;
11360	struct notifier_block *nb;
11361	int err;
11362
11363	router = kzalloc(size: sizeof(*mlxsw_sp->router), GFP_KERNEL);
11364	if (!router)
11365	return -ENOMEM;
11366	mutex_init(&router->lock);
11367	mlxsw_sp->router = router;
11368	router->mlxsw_sp = mlxsw_sp;
11369
11370	err = mlxsw_sp->router_ops->init(mlxsw_sp);
11371	if (err)
11372	goto err_router_ops_init;
11373
11374	INIT_LIST_HEAD(list: &mlxsw_sp->router->nh_res_grp_list);
11375	INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw,
11376	mlxsw_sp_nh_grp_activity_work);
11377	INIT_LIST_HEAD(list: &mlxsw_sp->router->nexthop_neighs_list);
11378	err = __mlxsw_sp_router_init(mlxsw_sp);
11379	if (err)
11380	goto err_router_init;
11381
11382	err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp);
11383	if (err)
11384	goto err_ipips_init;
11385
11386	err = rhashtable_init(ht: &mlxsw_sp->router->crif_ht,
11387	params: &mlxsw_sp_crif_ht_params);
11388	if (err)
11389	goto err_crif_ht_init;
11390
11391	err = mlxsw_sp_rifs_init(mlxsw_sp);
11392	if (err)
11393	goto err_rifs_init;
11394
11395	err = rhashtable_init(ht: &mlxsw_sp->router->nexthop_ht,
11396	params: &mlxsw_sp_nexthop_ht_params);
11397	if (err)
11398	goto err_nexthop_ht_init;
11399
11400	err = rhashtable_init(ht: &mlxsw_sp->router->nexthop_group_ht,
11401	params: &mlxsw_sp_nexthop_group_ht_params);
11402	if (err)
11403	goto err_nexthop_group_ht_init;
11404
11405	INIT_LIST_HEAD(list: &mlxsw_sp->router->nexthop_list);
11406	err = mlxsw_sp_lpm_init(mlxsw_sp);
11407	if (err)
11408	goto err_lpm_init;
11409
11410	err = mlxsw_sp_mr_init(mlxsw_sp, mr_ops: &mlxsw_sp_mr_tcam_ops);
11411	if (err)
11412	goto err_mr_init;
11413
11414	err = mlxsw_sp_vrs_init(mlxsw_sp);
11415	if (err)
11416	goto err_vrs_init;
11417
11418	err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack);
11419	if (err)
11420	goto err_lb_rif_init;
11421
11422	err = mlxsw_sp_neigh_init(mlxsw_sp);
11423	if (err)
11424	goto err_neigh_init;
11425
11426	err = mlxsw_sp_mp_hash_init(mlxsw_sp);
11427	if (err)
11428	goto err_mp_hash_init;
11429
11430	err = mlxsw_sp_dscp_init(mlxsw_sp);
11431	if (err)
11432	goto err_dscp_init;
11433
11434	router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
11435	err = register_inetaddr_notifier(nb: &router->inetaddr_nb);
11436	if (err)
11437	goto err_register_inetaddr_notifier;
11438
11439	router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
11440	err = register_inet6addr_notifier(nb: &router->inet6addr_nb);
11441	if (err)
11442	goto err_register_inet6addr_notifier;
11443
11444	router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event;
11445	err = register_inetaddr_validator_notifier(nb: &router->inetaddr_valid_nb);
11446	if (err)
11447	goto err_register_inetaddr_valid_notifier;
11448
11449	nb = &router->inet6addr_valid_nb;
11450	nb->notifier_call = mlxsw_sp_inet6addr_valid_event;
11451	err = register_inet6addr_validator_notifier(nb);
11452	if (err)
11453	goto err_register_inet6addr_valid_notifier;
11454
11455	mlxsw_sp->router->netevent_nb.notifier_call =
11456	mlxsw_sp_router_netevent_event;
11457	err = register_netevent_notifier(nb: &mlxsw_sp->router->netevent_nb);
11458	if (err)
11459	goto err_register_netevent_notifier;
11460
11461	mlxsw_sp->router->nexthop_nb.notifier_call =
11462	mlxsw_sp_nexthop_obj_event;
11463	err = register_nexthop_notifier(net: mlxsw_sp_net(mlxsw_sp),
11464	nb: &mlxsw_sp->router->nexthop_nb,
11465	extack);
11466	if (err)
11467	goto err_register_nexthop_notifier;
11468
11469	mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
11470	err = register_fib_notifier(net: mlxsw_sp_net(mlxsw_sp),
11471	nb: &mlxsw_sp->router->fib_nb,
11472	cb: mlxsw_sp_router_fib_dump_flush, extack);
11473	if (err)
11474	goto err_register_fib_notifier;
11475
11476	mlxsw_sp->router->netdevice_nb.notifier_call =
11477	mlxsw_sp_router_netdevice_event;
11478	err = register_netdevice_notifier_net(net: mlxsw_sp_net(mlxsw_sp),
11479	nb: &mlxsw_sp->router->netdevice_nb);
11480	if (err)
11481	goto err_register_netdev_notifier;
11482
11483	return 0;
11484
11485	err_register_netdev_notifier:
11486	unregister_fib_notifier(net: mlxsw_sp_net(mlxsw_sp),
11487	nb: &mlxsw_sp->router->fib_nb);
11488	err_register_fib_notifier:
11489	unregister_nexthop_notifier(net: mlxsw_sp_net(mlxsw_sp),
11490	nb: &mlxsw_sp->router->nexthop_nb);
11491	err_register_nexthop_notifier:
11492	unregister_netevent_notifier(nb: &mlxsw_sp->router->netevent_nb);
11493	err_register_netevent_notifier:
11494	unregister_inet6addr_validator_notifier(nb: &router->inet6addr_valid_nb);
11495	err_register_inet6addr_valid_notifier:
11496	unregister_inetaddr_validator_notifier(nb: &router->inetaddr_valid_nb);
11497	err_register_inetaddr_valid_notifier:
11498	unregister_inet6addr_notifier(nb: &router->inet6addr_nb);
11499	err_register_inet6addr_notifier:
11500	unregister_inetaddr_notifier(nb: &router->inetaddr_nb);
11501	err_register_inetaddr_notifier:
11502	mlxsw_core_flush_owq();
11503	err_dscp_init:
11504	mlxsw_sp_mp_hash_fini(mlxsw_sp);
11505	err_mp_hash_init:
11506	mlxsw_sp_neigh_fini(mlxsw_sp);
11507	err_neigh_init:
11508	mlxsw_sp_lb_rif_fini(mlxsw_sp);
11509	err_lb_rif_init:
11510	mlxsw_sp_vrs_fini(mlxsw_sp);
11511	err_vrs_init:
11512	mlxsw_sp_mr_fini(mlxsw_sp);
11513	err_mr_init:
11514	mlxsw_sp_lpm_fini(mlxsw_sp);
11515	err_lpm_init:
11516	rhashtable_destroy(ht: &mlxsw_sp->router->nexthop_group_ht);
11517	err_nexthop_group_ht_init:
11518	rhashtable_destroy(ht: &mlxsw_sp->router->nexthop_ht);
11519	err_nexthop_ht_init:
11520	mlxsw_sp_rifs_fini(mlxsw_sp);
11521	err_rifs_init:
11522	rhashtable_destroy(ht: &mlxsw_sp->router->crif_ht);
11523	err_crif_ht_init:
11524	mlxsw_sp_ipips_fini(mlxsw_sp);
11525	err_ipips_init:
11526	__mlxsw_sp_router_fini(mlxsw_sp);
11527	err_router_init:
11528	cancel_delayed_work_sync(dwork: &mlxsw_sp->router->nh_grp_activity_dw);
11529	err_router_ops_init:
11530	mutex_destroy(lock: &mlxsw_sp->router->lock);
11531	kfree(objp: mlxsw_sp->router);
11532	return err;
11533	}
11534
11535	void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
11536	{
11537	struct mlxsw_sp_router *router = mlxsw_sp->router;
11538
11539	unregister_netdevice_notifier_net(net: mlxsw_sp_net(mlxsw_sp),
11540	nb: &router->netdevice_nb);
11541	unregister_fib_notifier(net: mlxsw_sp_net(mlxsw_sp), nb: &router->fib_nb);
11542	unregister_nexthop_notifier(net: mlxsw_sp_net(mlxsw_sp),
11543	nb: &router->nexthop_nb);
11544	unregister_netevent_notifier(nb: &router->netevent_nb);
11545	unregister_inet6addr_validator_notifier(nb: &router->inet6addr_valid_nb);
11546	unregister_inetaddr_validator_notifier(nb: &router->inetaddr_valid_nb);
11547	unregister_inet6addr_notifier(nb: &router->inet6addr_nb);
11548	unregister_inetaddr_notifier(nb: &router->inetaddr_nb);
11549	mlxsw_core_flush_owq();
11550	mlxsw_sp_mp_hash_fini(mlxsw_sp);
11551	mlxsw_sp_neigh_fini(mlxsw_sp);
11552	mlxsw_sp_lb_rif_fini(mlxsw_sp);
11553	mlxsw_sp_vrs_fini(mlxsw_sp);
11554	mlxsw_sp_mr_fini(mlxsw_sp);
11555	mlxsw_sp_lpm_fini(mlxsw_sp);
11556	rhashtable_destroy(ht: &router->nexthop_group_ht);
11557	rhashtable_destroy(ht: &router->nexthop_ht);
11558	mlxsw_sp_rifs_fini(mlxsw_sp);
11559	rhashtable_destroy(ht: &mlxsw_sp->router->crif_ht);
11560	mlxsw_sp_ipips_fini(mlxsw_sp);
11561	__mlxsw_sp_router_fini(mlxsw_sp);
11562	cancel_delayed_work_sync(dwork: &router->nh_grp_activity_dw);
11563	mutex_destroy(lock: &router->lock);
11564	kfree(objp: router);
11565	}
11566