1	/*
2	* Copyright (c) 2004 Topspin Communications. All rights reserved.
3	* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4	* Copyright (c) 2004 Voltaire, Inc. All rights reserved.
5	*
6	* This software is available to you under a choice of one of two
7	* licenses. You may choose to be licensed under the terms of the GNU
8	* General Public License (GPL) Version 2, available from the file
9	* COPYING in the main directory of this source tree, or the
10	* OpenIB.org BSD license below:
11	*
12	* Redistribution and use in source and binary forms, with or
13	* without modification, are permitted provided that the following
14	* conditions are met:
15	*
16	* - Redistributions of source code must retain the above
17	* copyright notice, this list of conditions and the following
18	* disclaimer.
19	*
20	* - Redistributions in binary form must reproduce the above
21	* copyright notice, this list of conditions and the following
22	* disclaimer in the documentation and/or other materials
23	* provided with the distribution.
24	*
25	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32	* SOFTWARE.
33	*/
34
35	#include "ipoib.h"
36
37	#include <linux/module.h>
38
39	#include <linux/init.h>
40	#include <linux/slab.h>
41	#include <linux/kernel.h>
42	#include <linux/vmalloc.h>
43
44	#include <linux/if_arp.h> /* For ARPHRD_xxx */
45
46	#include <linux/ip.h>
47	#include <linux/in.h>
48
49	#include <linux/jhash.h>
50	#include <net/arp.h>
51	#include <net/addrconf.h>
52	#include <net/netdev_lock.h>
53	#include <net/pkt_sched.h>
54	#include <linux/inetdevice.h>
55	#include <rdma/ib_cache.h>
56
57	MODULE_AUTHOR("Roland Dreier");
58	MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
59	MODULE_LICENSE("Dual BSD/GPL");
60
61	int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
62	int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;
63
64	module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);
65	MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
66	module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
67	MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");
68
69	#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
70	int ipoib_debug_level;
71
72	module_param_named(debug_level, ipoib_debug_level, int, 0644);
73	MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
74	#endif
75
76	struct ipoib_path_iter {
77	struct net_device *dev;
78	struct ipoib_path path;
79	};
80
81	static const u8 ipv4_bcast_addr[] = {
82	0x00, 0xff, 0xff, 0xff,
83	0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
84	0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
85	};
86
87	struct workqueue_struct *ipoib_workqueue;
88
89	struct ib_sa_client ipoib_sa_client;
90
91	static int ipoib_add_one(struct ib_device *device);
92	static void ipoib_remove_one(struct ib_device *device, void *client_data);
93	static void ipoib_neigh_reclaim(struct rcu_head *rp);
94	static struct net_device *ipoib_get_net_dev_by_params(
95	struct ib_device *dev, u32 port, u16 pkey,
96	const union ib_gid *gid, const struct sockaddr *addr,
97	void *client_data);
98	static int ipoib_set_mac(struct net_device *dev, void *addr);
99	static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
100	int cmd);
101
102	static struct ib_client ipoib_client = {
103	.name = "ipoib",
104	.add = ipoib_add_one,
105	.remove = ipoib_remove_one,
106	.get_net_dev_by_params = ipoib_get_net_dev_by_params,
107	};
108
109	#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
110	static int ipoib_netdev_event(struct notifier_block *this,
111	unsigned long event, void *ptr)
112	{
113	struct netdev_notifier_info *ni = ptr;
114	struct net_device *dev = ni->dev;
115
116	if (dev->netdev_ops->ndo_open != ipoib_open)
117	return NOTIFY_DONE;
118
119	switch (event) {
120	case NETDEV_REGISTER:
121	ipoib_create_debug_files(dev);
122	break;
123	case NETDEV_CHANGENAME:
124	ipoib_delete_debug_files(dev);
125	ipoib_create_debug_files(dev);
126	break;
127	case NETDEV_UNREGISTER:
128	ipoib_delete_debug_files(dev);
129	break;
130	}
131
132	return NOTIFY_DONE;
133	}
134	#endif
135
136	struct ipoib_ifupdown_work {
137	struct work_struct work;
138	struct net_device *dev;
139	netdevice_tracker dev_tracker;
140	bool up;
141	};
142
143	static void ipoib_ifupdown_task(struct work_struct *work)
144	{
145	struct ipoib_ifupdown_work *pwork =
146	container_of(work, struct ipoib_ifupdown_work, work);
147	struct net_device *dev = pwork->dev;
148	unsigned int flags;
149
150	rtnl_lock();
151	flags = dev->flags;
152	if (pwork->up)
153	flags |= IFF_UP;
154	else
155	flags &= ~IFF_UP;
156
157	if (dev->flags != flags)
158	dev_change_flags(dev, flags, NULL);
159	rtnl_unlock();
160	netdev_put(dev, tracker: &pwork->dev_tracker);
161	kfree(objp: pwork);
162	}
163
164	static void ipoib_schedule_ifupdown_task(struct net_device *dev, bool up)
165	{
166	struct ipoib_ifupdown_work *work;
167
168	if ((up && (dev->flags & IFF_UP)) ||
169	(!up && !(dev->flags & IFF_UP)))
170	return;
171
172	work = kmalloc(sizeof(*work), GFP_KERNEL);
173	if (!work)
174	return;
175	work->dev = dev;
176	netdev_hold(dev, tracker: &work->dev_tracker, GFP_KERNEL);
177	work->up = up;
178	INIT_WORK(&work->work, ipoib_ifupdown_task);
179	queue_work(wq: ipoib_workqueue, work: &work->work);
180	}
181
182	int ipoib_open(struct net_device *dev)
183	{
184	struct ipoib_dev_priv *priv = ipoib_priv(dev);
185
186	ipoib_dbg(priv, "bringing up interface\n");
187
188	netif_carrier_off(dev);
189
190	set_bit(nr: IPOIB_FLAG_ADMIN_UP, addr: &priv->flags);
191
192	if (ipoib_ib_dev_open(dev)) {
193	if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
194	return 0;
195	goto err_disable;
196	}
197
198	ipoib_ib_dev_up(dev);
199
200	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
201	struct ipoib_dev_priv *cpriv;
202
203	/* Bring up any child interfaces too */
204	netdev_lock_ops_to_full(dev);
205	list_for_each_entry(cpriv, &priv->child_intfs, list)
206	ipoib_schedule_ifupdown_task(dev: cpriv->dev, up: true);
207	netdev_unlock_full_to_ops(dev);
208	} else if (priv->parent) {
209	struct ipoib_dev_priv *ppriv = ipoib_priv(dev: priv->parent);
210
211	if (!test_bit(IPOIB_FLAG_ADMIN_UP, &ppriv->flags))
212	ipoib_dbg(priv, "parent device %s is not up, so child device may be not functioning.\n",
213	ppriv->dev->name);
214	}
215	netif_start_queue(dev);
216
217	return 0;
218
219	err_disable:
220	clear_bit(nr: IPOIB_FLAG_ADMIN_UP, addr: &priv->flags);
221
222	return -EINVAL;
223	}
224
225	static int ipoib_stop(struct net_device *dev)
226	{
227	struct ipoib_dev_priv *priv = ipoib_priv(dev);
228
229	ipoib_dbg(priv, "stopping interface\n");
230
231	clear_bit(nr: IPOIB_FLAG_ADMIN_UP, addr: &priv->flags);
232
233	netif_stop_queue(dev);
234
235	ipoib_ib_dev_down(dev);
236	ipoib_ib_dev_stop(dev);
237
238	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
239	struct ipoib_dev_priv *cpriv;
240
241	/* Bring down any child interfaces too */
242	netdev_lock_ops_to_full(dev);
243	list_for_each_entry(cpriv, &priv->child_intfs, list)
244	ipoib_schedule_ifupdown_task(dev: cpriv->dev, up: false);
245	netdev_unlock_full_to_ops(dev);
246	}
247
248	return 0;
249	}
250
251	static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)
252	{
253	struct ipoib_dev_priv *priv = ipoib_priv(dev);
254
255	if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
256	features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);
257
258	return features;
259	}
260
261	static int ipoib_change_mtu(struct net_device *dev, int new_mtu)
262	{
263	struct ipoib_dev_priv *priv = ipoib_priv(dev);
264	int ret = 0;
265
266	/* dev->mtu > 2K ==> connected mode */
267	if (ipoib_cm_admin_enabled(dev)) {
268	if (new_mtu > ipoib_cm_max_mtu(dev))
269	return -EINVAL;
270
271	if (new_mtu > priv->mcast_mtu)
272	ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
273	priv->mcast_mtu);
274
275	WRITE_ONCE(dev->mtu, new_mtu);
276	return 0;
277	}
278
279	if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||
280	new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
281	return -EINVAL;
282
283	priv->admin_mtu = new_mtu;
284
285	if (priv->mcast_mtu < priv->admin_mtu)
286	ipoib_dbg(priv, "MTU must be smaller than the underlying "
287	"link layer MTU - 4 (%u)\n", priv->mcast_mtu);
288
289	new_mtu = min(priv->mcast_mtu, priv->admin_mtu);
290
291	if (priv->rn_ops->ndo_change_mtu) {
292	bool carrier_status = netif_carrier_ok(dev);
293
294	netif_carrier_off(dev);
295
296	/* notify lower level on the real mtu */
297	ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);
298
299	if (carrier_status)
300	netif_carrier_on(dev);
301	} else {
302	WRITE_ONCE(dev->mtu, new_mtu);
303	}
304
305	return ret;
306	}
307
308	static void ipoib_get_stats(struct net_device *dev,
309	struct rtnl_link_stats64 *stats)
310	{
311	struct ipoib_dev_priv *priv = ipoib_priv(dev);
312
313	if (priv->rn_ops->ndo_get_stats64)
314	priv->rn_ops->ndo_get_stats64(dev, stats);
315	else
316	netdev_stats_to_stats64(stats64: stats, netdev_stats: &dev->stats);
317	}
318
319	/* Called with an RCU read lock taken */
320	static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,
321	struct net_device *dev)
322	{
323	struct net *net = dev_net(dev);
324	struct in_device *in_dev;
325	struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;
326	struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;
327	__be32 ret_addr;
328
329	switch (addr->sa_family) {
330	case AF_INET:
331	in_dev = in_dev_get(dev);
332	if (!in_dev)
333	return false;
334
335	ret_addr = inet_confirm_addr(net, in_dev, dst: 0,
336	local: addr_in->sin_addr.s_addr,
337	scope: RT_SCOPE_HOST);
338	in_dev_put(idev: in_dev);
339	if (ret_addr)
340	return true;
341
342	break;
343	case AF_INET6:
344	if (IS_ENABLED(CONFIG_IPV6) &&
345	ipv6_chk_addr(net, addr: &addr_in6->sin6_addr, dev, strict: 1))
346	return true;
347
348	break;
349	}
350	return false;
351	}
352
353	/*
354	* Find the L2 master net_device on top of the given net_device.
355	* @dev: base IPoIB net_device
356	*
357	* Returns the L2 master net_device with reference held if the L2 master
358	* exists (such as bond netdevice), or returns same netdev with reference
359	* held when master does not exist or when L3 master (such as VRF netdev).
360	*/
361	static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)
362	{
363	struct net_device *master;
364
365	rcu_read_lock();
366
367	master = netdev_master_upper_dev_get_rcu(dev);
368	if (!master || netif_is_l3_master(dev: master))
369	master = dev;
370
371	dev_hold(dev: master);
372	rcu_read_unlock();
373
374	return master;
375	}
376
377	struct ipoib_walk_data {
378	const struct sockaddr *addr;
379	struct net_device *result;
380	};
381
382	static int ipoib_upper_walk(struct net_device *upper,
383	struct netdev_nested_priv *priv)
384	{
385	struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;
386	int ret = 0;
387
388	if (ipoib_is_dev_match_addr_rcu(addr: data->addr, dev: upper)) {
389	dev_hold(dev: upper);
390	data->result = upper;
391	ret = 1;
392	}
393
394	return ret;
395	}
396
397	/**
398	* ipoib_get_net_dev_match_addr - Find a net_device matching
399	* the given address, which is an upper device of the given net_device.
400	*
401	* @addr: IP address to look for.
402	* @dev: base IPoIB net_device
403	*
404	* If found, returns the net_device with a reference held. Otherwise return
405	* NULL.
406	*/
407	static struct net_device *ipoib_get_net_dev_match_addr(
408	const struct sockaddr *addr, struct net_device *dev)
409	{
410	struct netdev_nested_priv priv;
411	struct ipoib_walk_data data = {
412	.addr = addr,
413	};
414
415	priv.data = (void *)&data;
416	rcu_read_lock();
417	if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
418	dev_hold(dev);
419	data.result = dev;
420	goto out;
421	}
422
423	netdev_walk_all_upper_dev_rcu(dev, fn: ipoib_upper_walk, priv: &priv);
424	out:
425	rcu_read_unlock();
426	return data.result;
427	}
428
429	/* returns the number of IPoIB netdevs on top a given ipoib device matching a
430	* pkey_index and address, if one exists.
431	*
432	* @found_net_dev: contains a matching net_device if the return value >= 1,
433	* with a reference held. */
434	static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,
435	const union ib_gid *gid,
436	u16 pkey_index,
437	const struct sockaddr *addr,
438	int nesting,
439	struct net_device **found_net_dev)
440	{
441	struct ipoib_dev_priv *child_priv;
442	struct net_device *net_dev = NULL;
443	int matches = 0;
444
445	if (priv->pkey_index == pkey_index &&
446	(!gid || !memcmp(p: gid, q: &priv->local_gid, size: sizeof(*gid)))) {
447	if (!addr) {
448	net_dev = ipoib_get_master_net_dev(dev: priv->dev);
449	} else {
450	/* Verify the net_device matches the IP address, as
451	* IPoIB child devices currently share a GID. */
452	net_dev = ipoib_get_net_dev_match_addr(addr, dev: priv->dev);
453	}
454	if (net_dev) {
455	if (!*found_net_dev)
456	*found_net_dev = net_dev;
457	else
458	dev_put(dev: net_dev);
459	++matches;
460	}
461	}
462
463	if (test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
464	return matches;
465
466	/* Check child interfaces */
467	netdev_lock(dev: priv->dev);
468	list_for_each_entry(child_priv, &priv->child_intfs, list) {
469	matches += ipoib_match_gid_pkey_addr(priv: child_priv, gid,
470	pkey_index, addr,
471	nesting: nesting + 1,
472	found_net_dev);
473	if (matches > 1)
474	break;
475	}
476	netdev_unlock(dev: priv->dev);
477
478	return matches;
479	}
480
481	/* Returns the number of matching net_devs found (between 0 and 2). Also
482	* return the matching net_device in the @net_dev parameter, holding a
483	* reference to the net_device, if the number of matches >= 1 */
484	static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u32 port,
485	u16 pkey_index,
486	const union ib_gid *gid,
487	const struct sockaddr *addr,
488	struct net_device **net_dev)
489	{
490	struct ipoib_dev_priv *priv;
491	int matches = 0;
492
493	*net_dev = NULL;
494
495	list_for_each_entry(priv, dev_list, list) {
496	if (priv->port != port)
497	continue;
498
499	matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,
500	addr, nesting: 0, found_net_dev: net_dev);
501	if (matches > 1)
502	break;
503	}
504
505	return matches;
506	}
507
508	static struct net_device *ipoib_get_net_dev_by_params(
509	struct ib_device *dev, u32 port, u16 pkey,
510	const union ib_gid *gid, const struct sockaddr *addr,
511	void *client_data)
512	{
513	struct net_device *net_dev;
514	struct list_head *dev_list = client_data;
515	u16 pkey_index;
516	int matches;
517	int ret;
518
519	if (!rdma_protocol_ib(device: dev, port_num: port))
520	return NULL;
521
522	ret = ib_find_cached_pkey(device: dev, port_num: port, pkey, index: &pkey_index);
523	if (ret)
524	return NULL;
525
526	/* See if we can find a unique device matching the pkey and GID */
527	matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
528	gid, NULL, net_dev: &net_dev);
529
530	switch (matches) {
531	case 0:
532	return NULL;
533	case 1:
534	return net_dev;
535	}
536
537	dev_put(dev: net_dev);
538
539	/* Couldn't find a unique device with pkey and GID only. Use L3
540	* address to uniquely match the net device */
541	matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
542	gid, addr, net_dev: &net_dev);
543	switch (matches) {
544	case 0:
545	return NULL;
546	default:
547	dev_warn_ratelimited(&dev->dev,
548	"duplicate IP address detected\n");
549	fallthrough;
550	case 1:
551	return net_dev;
552	}
553	}
554
555	int ipoib_set_mode(struct net_device *dev, const char *buf)
556	{
557	struct ipoib_dev_priv *priv = ipoib_priv(dev);
558
559	if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
560	!strcmp(buf, "connected\n")) ||
561	(!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
562	!strcmp(buf, "datagram\n"))) {
563	return 0;
564	}
565
566	/* flush paths if we switch modes so that connections are restarted */
567	if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {
568	set_bit(nr: IPOIB_FLAG_ADMIN_CM, addr: &priv->flags);
569	ipoib_warn(priv, "enabling connected mode "
570	"will cause multicast packet drops\n");
571	netdev_lock_ops(dev);
572	netdev_update_features(dev);
573	netif_set_mtu(dev, new_mtu: ipoib_cm_max_mtu(dev));
574	netif_set_real_num_tx_queues(dev, txq: 1);
575	netdev_unlock_ops(dev);
576	rtnl_unlock();
577	priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
578
579	ipoib_flush_paths(dev);
580	return (!rtnl_trylock()) ? -EBUSY : 0;
581	}
582
583	if (!strcmp(buf, "datagram\n")) {
584	clear_bit(nr: IPOIB_FLAG_ADMIN_CM, addr: &priv->flags);
585	netdev_lock_ops(dev);
586	netdev_update_features(dev);
587	netif_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
588	netif_set_real_num_tx_queues(dev, txq: dev->num_tx_queues);
589	netdev_unlock_ops(dev);
590	rtnl_unlock();
591	ipoib_flush_paths(dev);
592	return (!rtnl_trylock()) ? -EBUSY : 0;
593	}
594
595	return -EINVAL;
596	}
597
598	struct ipoib_path *__path_find(struct net_device *dev, void *gid)
599	{
600	struct ipoib_dev_priv *priv = ipoib_priv(dev);
601	struct rb_node *n = priv->path_tree.rb_node;
602	struct ipoib_path *path;
603	int ret;
604
605	while (n) {
606	path = rb_entry(n, struct ipoib_path, rb_node);
607
608	ret = memcmp(p: gid, q: path->pathrec.dgid.raw,
609	size: sizeof (union ib_gid));
610
611	if (ret < 0)
612	n = n->rb_left;
613	else if (ret > 0)
614	n = n->rb_right;
615	else
616	return path;
617	}
618
619	return NULL;
620	}
621
622	static int __path_add(struct net_device *dev, struct ipoib_path *path)
623	{
624	struct ipoib_dev_priv *priv = ipoib_priv(dev);
625	struct rb_node **n = &priv->path_tree.rb_node;
626	struct rb_node *pn = NULL;
627	struct ipoib_path *tpath;
628	int ret;
629
630	while (*n) {
631	pn = *n;
632	tpath = rb_entry(pn, struct ipoib_path, rb_node);
633
634	ret = memcmp(p: path->pathrec.dgid.raw, q: tpath->pathrec.dgid.raw,
635	size: sizeof (union ib_gid));
636	if (ret < 0)
637	n = &pn->rb_left;
638	else if (ret > 0)
639	n = &pn->rb_right;
640	else
641	return -EEXIST;
642	}
643
644	rb_link_node(node: &path->rb_node, parent: pn, rb_link: n);
645	rb_insert_color(&path->rb_node, &priv->path_tree);
646
647	list_add_tail(new: &path->list, head: &priv->path_list);
648
649	return 0;
650	}
651
652	static void path_free(struct net_device *dev, struct ipoib_path *path)
653	{
654	struct sk_buff *skb;
655
656	while ((skb = __skb_dequeue(list: &path->queue)))
657	dev_kfree_skb_irq(skb);
658
659	ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);
660
661	/* remove all neigh connected to this path */
662	ipoib_del_neighs_by_gid(dev, gid: path->pathrec.dgid.raw);
663
664	if (path->ah)
665	ipoib_put_ah(ah: path->ah);
666
667	kfree(objp: path);
668	}
669
670	#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
671
672	struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)
673	{
674	struct ipoib_path_iter *iter;
675
676	iter = kmalloc(sizeof(*iter), GFP_KERNEL);
677	if (!iter)
678	return NULL;
679
680	iter->dev = dev;
681	memset(iter->path.pathrec.dgid.raw, 0, 16);
682
683	if (ipoib_path_iter_next(iter)) {
684	kfree(objp: iter);
685	return NULL;
686	}
687
688	return iter;
689	}
690
691	int ipoib_path_iter_next(struct ipoib_path_iter *iter)
692	{
693	struct ipoib_dev_priv *priv = ipoib_priv(dev: iter->dev);
694	struct rb_node *n;
695	struct ipoib_path *path;
696	int ret = 1;
697
698	spin_lock_irq(lock: &priv->lock);
699
700	n = rb_first(root: &priv->path_tree);
701
702	while (n) {
703	path = rb_entry(n, struct ipoib_path, rb_node);
704
705	if (memcmp(p: iter->path.pathrec.dgid.raw, q: path->pathrec.dgid.raw,
706	size: sizeof (union ib_gid)) < 0) {
707	iter->path = *path;
708	ret = 0;
709	break;
710	}
711
712	n = rb_next(n);
713	}
714
715	spin_unlock_irq(lock: &priv->lock);
716
717	return ret;
718	}
719
720	void ipoib_path_iter_read(struct ipoib_path_iter *iter,
721	struct ipoib_path *path)
722	{
723	*path = iter->path;
724	}
725
726	#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */
727
728	void ipoib_mark_paths_invalid(struct net_device *dev)
729	{
730	struct ipoib_dev_priv *priv = ipoib_priv(dev);
731	struct ipoib_path *path, *tp;
732
733	spin_lock_irq(lock: &priv->lock);
734
735	list_for_each_entry_safe(path, tp, &priv->path_list, list) {
736	ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
737	be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
738	path->pathrec.dgid.raw);
739	if (path->ah)
740	path->ah->valid = 0;
741	}
742
743	spin_unlock_irq(lock: &priv->lock);
744	}
745
746	static void push_pseudo_header(struct sk_buff *skb, const char *daddr)
747	{
748	struct ipoib_pseudo_header *phdr;
749
750	phdr = skb_push(skb, len: sizeof(*phdr));
751	memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);
752	}
753
754	void ipoib_flush_paths(struct net_device *dev)
755	{
756	struct ipoib_dev_priv *priv = ipoib_priv(dev);
757	struct ipoib_path *path, *tp;
758	LIST_HEAD(remove_list);
759	unsigned long flags;
760
761	netif_tx_lock_bh(dev);
762	spin_lock_irqsave(&priv->lock, flags);
763
764	list_splice_init(list: &priv->path_list, head: &remove_list);
765
766	list_for_each_entry(path, &remove_list, list)
767	rb_erase(&path->rb_node, &priv->path_tree);
768
769	list_for_each_entry_safe(path, tp, &remove_list, list) {
770	if (path->query)
771	ib_sa_cancel_query(id: path->query_id, query: path->query);
772	spin_unlock_irqrestore(lock: &priv->lock, flags);
773	netif_tx_unlock_bh(dev);
774	wait_for_completion(&path->done);
775	path_free(dev, path);
776	netif_tx_lock_bh(dev);
777	spin_lock_irqsave(&priv->lock, flags);
778	}
779
780	spin_unlock_irqrestore(lock: &priv->lock, flags);
781	netif_tx_unlock_bh(dev);
782	}
783
784	static void path_rec_completion(int status,
785	struct sa_path_rec *pathrec,
786	unsigned int num_prs, void *path_ptr)
787	{
788	struct ipoib_path *path = path_ptr;
789	struct net_device *dev = path->dev;
790	struct ipoib_dev_priv *priv = ipoib_priv(dev);
791	struct ipoib_ah *ah = NULL;
792	struct ipoib_ah *old_ah = NULL;
793	struct ipoib_neigh *neigh, *tn;
794	struct sk_buff_head skqueue;
795	struct sk_buff *skb;
796	unsigned long flags;
797
798	if (!status)
799	ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",
800	be32_to_cpu(sa_path_get_dlid(pathrec)),
801	pathrec->dgid.raw);
802	else
803	ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",
804	status, path->pathrec.dgid.raw);
805
806	skb_queue_head_init(list: &skqueue);
807
808	if (!status) {
809	struct rdma_ah_attr av;
810
811	if (!ib_init_ah_attr_from_path(device: priv->ca, port_num: priv->port,
812	rec: pathrec, ah_attr: &av, NULL)) {
813	ah = ipoib_create_ah(dev, pd: priv->pd, attr: &av);
814	rdma_destroy_ah_attr(ah_attr: &av);
815	}
816	}
817
818	spin_lock_irqsave(&priv->lock, flags);
819
820	if (!IS_ERR_OR_NULL(ptr: ah)) {
821	/*
822	* pathrec.dgid is used as the database key from the LLADDR,
823	* it must remain unchanged even if the SA returns a different
824	* GID to use in the AH.
825	*/
826	if (memcmp(p: pathrec->dgid.raw, q: path->pathrec.dgid.raw,
827	size: sizeof(union ib_gid))) {
828	ipoib_dbg(
829	priv,
830	"%s got PathRec for gid %pI6 while asked for %pI6\n",
831	dev->name, pathrec->dgid.raw,
832	path->pathrec.dgid.raw);
833	memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
834	sizeof(union ib_gid));
835	}
836
837	path->pathrec = *pathrec;
838
839	old_ah = path->ah;
840	path->ah = ah;
841
842	ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",
843	ah, be32_to_cpu(sa_path_get_dlid(pathrec)),
844	pathrec->sl);
845
846	while ((skb = __skb_dequeue(list: &path->queue)))
847	__skb_queue_tail(list: &skqueue, newsk: skb);
848
849	list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
850	if (neigh->ah) {
851	WARN_ON(neigh->ah != old_ah);
852	/*
853	* Dropping the ah reference inside
854	* priv->lock is safe here, because we
855	* will hold one more reference from
856	* the original value of path->ah (ie
857	* old_ah).
858	*/
859	ipoib_put_ah(ah: neigh->ah);
860	}
861	kref_get(kref: &path->ah->ref);
862	neigh->ah = path->ah;
863
864	if (ipoib_cm_enabled(dev, hwaddr: neigh->daddr)) {
865	if (!ipoib_cm_get(neigh))
866	ipoib_cm_set(neigh, tx: ipoib_cm_create_tx(dev,
867	path,
868	neigh));
869	if (!ipoib_cm_get(neigh)) {
870	ipoib_neigh_free(neigh);
871	continue;
872	}
873	}
874
875	while ((skb = __skb_dequeue(list: &neigh->queue)))
876	__skb_queue_tail(list: &skqueue, newsk: skb);
877	}
878	path->ah->valid = 1;
879	}
880
881	path->query = NULL;
882	complete(&path->done);
883
884	spin_unlock_irqrestore(lock: &priv->lock, flags);
885
886	if (IS_ERR_OR_NULL(ptr: ah))
887	ipoib_del_neighs_by_gid(dev, gid: path->pathrec.dgid.raw);
888
889	if (old_ah)
890	ipoib_put_ah(ah: old_ah);
891
892	while ((skb = __skb_dequeue(list: &skqueue))) {
893	int ret;
894	skb->dev = dev;
895	ret = dev_queue_xmit(skb);
896	if (ret)
897	ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",
898	__func__, ret);
899	}
900	}
901
902	static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
903	void *gid)
904	{
905	path->dev = priv->dev;
906
907	if (rdma_cap_opa_ah(device: priv->ca, port_num: priv->port))
908	path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
909	else
910	path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;
911
912	memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
913	path->pathrec.sgid = priv->local_gid;
914	path->pathrec.pkey = cpu_to_be16(priv->pkey);
915	path->pathrec.numb_path = 1;
916	path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
917	}
918
919	static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
920	{
921	struct ipoib_dev_priv *priv = ipoib_priv(dev);
922	struct ipoib_path *path;
923
924	if (!priv->broadcast)
925	return NULL;
926
927	path = kzalloc(sizeof(*path), GFP_ATOMIC);
928	if (!path)
929	return NULL;
930
931	skb_queue_head_init(list: &path->queue);
932
933	INIT_LIST_HEAD(list: &path->neigh_list);
934
935	init_path_rec(priv, path, gid);
936
937	return path;
938	}
939
940	static int path_rec_start(struct net_device *dev,
941	struct ipoib_path *path)
942	{
943	struct ipoib_dev_priv *priv = ipoib_priv(dev);
944
945	ipoib_dbg(priv, "Start path record lookup for %pI6\n",
946	path->pathrec.dgid.raw);
947
948	init_completion(x: &path->done);
949
950	path->query_id =
951	ib_sa_path_rec_get(client: &ipoib_sa_client, device: priv->ca, port_num: priv->port,
952	rec: &path->pathrec,
953	IB_SA_PATH_REC_DGID |
954	IB_SA_PATH_REC_SGID |
955	IB_SA_PATH_REC_NUMB_PATH |
956	IB_SA_PATH_REC_TRAFFIC_CLASS |
957	IB_SA_PATH_REC_PKEY,
958	timeout_ms: 1000, GFP_ATOMIC,
959	callback: path_rec_completion,
960	context: path, query: &path->query);
961	if (path->query_id < 0) {
962	ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);
963	path->query = NULL;
964	complete(&path->done);
965	return path->query_id;
966	}
967
968	return 0;
969	}
970
971	static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
972	struct net_device *dev)
973	{
974	struct ipoib_dev_priv *priv = ipoib_priv(dev);
975	struct ipoib_path *path;
976	unsigned long flags;
977
978	spin_lock_irqsave(&priv->lock, flags);
979
980	path = __path_find(dev, gid: daddr + 4);
981	if (!path)
982	goto out;
983	if (!path->query)
984	path_rec_start(dev, path);
985	out:
986	spin_unlock_irqrestore(lock: &priv->lock, flags);
987	}
988
989	static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
990	struct net_device *dev)
991	{
992	struct ipoib_dev_priv *priv = ipoib_priv(dev);
993	struct rdma_netdev *rn = netdev_priv(dev);
994	struct ipoib_path *path;
995	struct ipoib_neigh *neigh;
996	unsigned long flags;
997
998	spin_lock_irqsave(&priv->lock, flags);
999	neigh = ipoib_neigh_alloc(daddr, dev);
1000	if (!neigh) {
1001	spin_unlock_irqrestore(lock: &priv->lock, flags);
1002	++dev->stats.tx_dropped;
1003	dev_kfree_skb_any(skb);
1004	return NULL;
1005	}
1006
1007	/* To avoid race condition, make sure that the
1008	* neigh will be added only once.
1009	*/
1010	if (unlikely(!list_empty(&neigh->list))) {
1011	spin_unlock_irqrestore(lock: &priv->lock, flags);
1012	return neigh;
1013	}
1014
1015	path = __path_find(dev, gid: daddr + 4);
1016	if (!path) {
1017	path = path_rec_create(dev, gid: daddr + 4);
1018	if (!path)
1019	goto err_path;
1020
1021	__path_add(dev, path);
1022	}
1023
1024	list_add_tail(new: &neigh->list, head: &path->neigh_list);
1025
1026	if (path->ah && path->ah->valid) {
1027	kref_get(kref: &path->ah->ref);
1028	neigh->ah = path->ah;
1029
1030	if (ipoib_cm_enabled(dev, hwaddr: neigh->daddr)) {
1031	if (!ipoib_cm_get(neigh))
1032	ipoib_cm_set(neigh, tx: ipoib_cm_create_tx(dev, path, neigh));
1033	if (!ipoib_cm_get(neigh)) {
1034	ipoib_neigh_free(neigh);
1035	goto err_drop;
1036	}
1037	if (skb_queue_len(list_: &neigh->queue) <
1038	IPOIB_MAX_PATH_REC_QUEUE) {
1039	push_pseudo_header(skb, daddr: neigh->daddr);
1040	__skb_queue_tail(list: &neigh->queue, newsk: skb);
1041	} else {
1042	ipoib_warn(priv, "queue length limit %d. Packet drop.\n",
1043	skb_queue_len(&neigh->queue));
1044	goto err_drop;
1045	}
1046	} else {
1047	spin_unlock_irqrestore(lock: &priv->lock, flags);
1048	path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1049	IPOIB_QPN(daddr));
1050	ipoib_neigh_put(neigh);
1051	return NULL;
1052	}
1053	} else {
1054	neigh->ah = NULL;
1055
1056	if (!path->query && path_rec_start(dev, path))
1057	goto err_path;
1058	if (skb_queue_len(list_: &neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1059	push_pseudo_header(skb, daddr: neigh->daddr);
1060	__skb_queue_tail(list: &neigh->queue, newsk: skb);
1061	} else {
1062	goto err_drop;
1063	}
1064	}
1065
1066	spin_unlock_irqrestore(lock: &priv->lock, flags);
1067	ipoib_neigh_put(neigh);
1068	return NULL;
1069
1070	err_path:
1071	ipoib_neigh_free(neigh);
1072	err_drop:
1073	++dev->stats.tx_dropped;
1074	dev_kfree_skb_any(skb);
1075
1076	spin_unlock_irqrestore(lock: &priv->lock, flags);
1077	ipoib_neigh_put(neigh);
1078
1079	return NULL;
1080	}
1081
1082	static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
1083	struct ipoib_pseudo_header *phdr)
1084	{
1085	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1086	struct rdma_netdev *rn = netdev_priv(dev);
1087	struct ipoib_path *path;
1088	unsigned long flags;
1089
1090	spin_lock_irqsave(&priv->lock, flags);
1091
1092	/* no broadcast means that all paths are (going to be) not valid */
1093	if (!priv->broadcast)
1094	goto drop_and_unlock;
1095
1096	path = __path_find(dev, gid: phdr->hwaddr + 4);
1097	if (!path || !path->ah || !path->ah->valid) {
1098	if (!path) {
1099	path = path_rec_create(dev, gid: phdr->hwaddr + 4);
1100	if (!path)
1101	goto drop_and_unlock;
1102	__path_add(dev, path);
1103	} else {
1104	/*
1105	* make sure there are no changes in the existing
1106	* path record
1107	*/
1108	init_path_rec(priv, path, gid: phdr->hwaddr + 4);
1109	}
1110	if (!path->query && path_rec_start(dev, path)) {
1111	goto drop_and_unlock;
1112	}
1113
1114	if (skb_queue_len(list_: &path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1115	push_pseudo_header(skb, daddr: phdr->hwaddr);
1116	__skb_queue_tail(list: &path->queue, newsk: skb);
1117	goto unlock;
1118	} else {
1119	goto drop_and_unlock;
1120	}
1121	}
1122
1123	spin_unlock_irqrestore(lock: &priv->lock, flags);
1124	ipoib_dbg(priv, "Send unicast ARP to %08x\n",
1125	be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
1126	path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1127	IPOIB_QPN(phdr->hwaddr));
1128	return;
1129
1130	drop_and_unlock:
1131	++dev->stats.tx_dropped;
1132	dev_kfree_skb_any(skb);
1133	unlock:
1134	spin_unlock_irqrestore(lock: &priv->lock, flags);
1135	}
1136
1137	static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
1138	{
1139	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1140	struct rdma_netdev *rn = netdev_priv(dev);
1141	struct ipoib_neigh *neigh;
1142	struct ipoib_pseudo_header *phdr;
1143	struct ipoib_header *header;
1144	unsigned long flags;
1145
1146	phdr = (struct ipoib_pseudo_header *) skb->data;
1147	skb_pull(skb, len: sizeof(*phdr));
1148	header = (struct ipoib_header *) skb->data;
1149
1150	if (unlikely(phdr->hwaddr[4] == 0xff)) {
1151	/* multicast, arrange "if" according to probability */
1152	if ((header->proto != htons(ETH_P_IP)) &&
1153	(header->proto != htons(ETH_P_IPV6)) &&
1154	(header->proto != htons(ETH_P_ARP)) &&
1155	(header->proto != htons(ETH_P_RARP)) &&
1156	(header->proto != htons(ETH_P_TIPC))) {
1157	/* ethertype not supported by IPoIB */
1158	++dev->stats.tx_dropped;
1159	dev_kfree_skb_any(skb);
1160	return NETDEV_TX_OK;
1161	}
1162	/* Add in the P_Key for multicast*/
1163	phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;
1164	phdr->hwaddr[9] = priv->pkey & 0xff;
1165
1166	neigh = ipoib_neigh_get(dev, daddr: phdr->hwaddr);
1167	if (likely(neigh))
1168	goto send_using_neigh;
1169	ipoib_mcast_send(dev, daddr: phdr->hwaddr, skb);
1170	return NETDEV_TX_OK;
1171	}
1172
1173	/* unicast, arrange "switch" according to probability */
1174	switch (header->proto) {
1175	case htons(ETH_P_IP):
1176	case htons(ETH_P_IPV6):
1177	case htons(ETH_P_TIPC):
1178	neigh = ipoib_neigh_get(dev, daddr: phdr->hwaddr);
1179	if (unlikely(!neigh)) {
1180	neigh = neigh_add_path(skb, daddr: phdr->hwaddr, dev);
1181	if (likely(!neigh))
1182	return NETDEV_TX_OK;
1183	}
1184	break;
1185	case htons(ETH_P_ARP):
1186	case htons(ETH_P_RARP):
1187	/* for unicast ARP and RARP should always perform path find */
1188	unicast_arp_send(skb, dev, phdr);
1189	return NETDEV_TX_OK;
1190	default:
1191	/* ethertype not supported by IPoIB */
1192	++dev->stats.tx_dropped;
1193	dev_kfree_skb_any(skb);
1194	return NETDEV_TX_OK;
1195	}
1196
1197	send_using_neigh:
1198	/* note we now hold a ref to neigh */
1199	if (ipoib_cm_get(neigh)) {
1200	if (ipoib_cm_up(neigh)) {
1201	ipoib_cm_send(dev, skb, tx: ipoib_cm_get(neigh));
1202	goto unref;
1203	}
1204	} else if (neigh->ah && neigh->ah->valid) {
1205	neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
1206	IPOIB_QPN(phdr->hwaddr));
1207	goto unref;
1208	} else if (neigh->ah) {
1209	neigh_refresh_path(neigh, daddr: phdr->hwaddr, dev);
1210	}
1211
1212	if (skb_queue_len(list_: &neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1213	push_pseudo_header(skb, daddr: phdr->hwaddr);
1214	spin_lock_irqsave(&priv->lock, flags);
1215	__skb_queue_tail(list: &neigh->queue, newsk: skb);
1216	spin_unlock_irqrestore(lock: &priv->lock, flags);
1217	} else {
1218	++dev->stats.tx_dropped;
1219	dev_kfree_skb_any(skb);
1220	}
1221
1222	unref:
1223	ipoib_neigh_put(neigh);
1224
1225	return NETDEV_TX_OK;
1226	}
1227
1228	static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
1229	{
1230	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1231	struct rdma_netdev *rn = netdev_priv(dev);
1232
1233	if (rn->tx_timeout) {
1234	rn->tx_timeout(dev, txqueue);
1235	return;
1236	}
1237	ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
1238	jiffies_to_msecs(jiffies - dev_trans_start(dev)));
1239	ipoib_warn(priv,
1240	"queue stopped %d, tx_head %u, tx_tail %u, global_tx_head %u, global_tx_tail %u\n",
1241	netif_queue_stopped(dev), priv->tx_head, priv->tx_tail,
1242	priv->global_tx_head, priv->global_tx_tail);
1243
1244
1245	schedule_work(work: &priv->tx_timeout_work);
1246	}
1247
1248	void ipoib_ib_tx_timeout_work(struct work_struct *work)
1249	{
1250	struct ipoib_dev_priv *priv = container_of(work,
1251	struct ipoib_dev_priv,
1252	tx_timeout_work);
1253	int err;
1254
1255	rtnl_lock();
1256	netdev_lock_ops(dev: priv->dev);
1257
1258	if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
1259	goto unlock;
1260
1261	ipoib_stop(dev: priv->dev);
1262	err = ipoib_open(dev: priv->dev);
1263	if (err) {
1264	ipoib_warn(priv, "ipoib_open failed recovering from a tx_timeout, err(%d).\n",
1265	err);
1266	goto unlock;
1267	}
1268
1269	netif_tx_wake_all_queues(dev: priv->dev);
1270	unlock:
1271	netdev_unlock_ops(dev: priv->dev);
1272	rtnl_unlock();
1273
1274	}
1275
1276	static int ipoib_hard_header(struct sk_buff *skb,
1277	struct net_device *dev,
1278	unsigned short type,
1279	const void *daddr,
1280	const void *saddr,
1281	unsigned int len)
1282	{
1283	struct ipoib_header *header;
1284
1285	header = skb_push(skb, len: sizeof(*header));
1286
1287	header->proto = htons(type);
1288	header->reserved = 0;
1289
1290	/*
1291	* we don't rely on dst_entry structure, always stuff the
1292	* destination address into skb hard header so we can figure out where
1293	* to send the packet later.
1294	*/
1295	push_pseudo_header(skb, daddr);
1296
1297	return IPOIB_HARD_LEN;
1298	}
1299
1300	static void ipoib_set_mcast_list(struct net_device *dev)
1301	{
1302	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1303
1304	if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
1305	ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
1306	return;
1307	}
1308
1309	queue_work(wq: priv->wq, work: &priv->restart_task);
1310	}
1311
1312	static int ipoib_get_iflink(const struct net_device *dev)
1313	{
1314	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1315
1316	/* parent interface */
1317	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
1318	return READ_ONCE(dev->ifindex);
1319
1320	/* child/vlan interface */
1321	return READ_ONCE(priv->parent->ifindex);
1322	}
1323
1324	static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)
1325	{
1326	/*
1327	* Use only the address parts that contributes to spreading
1328	* The subnet prefix is not used as one can not connect to
1329	* same remote port (GUID) using the same remote QPN via two
1330	* different subnets.
1331	*/
1332	/* qpn octets[1:4) & port GUID octets[12:20) */
1333	u32 *d32 = (u32 *) daddr;
1334	u32 hv;
1335
1336	hv = jhash_3words(a: d32[3], b: d32[4], IPOIB_QPN_MASK & d32[0], initval: 0);
1337	return hv & htbl->mask;
1338	}
1339
1340	struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
1341	{
1342	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1343	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1344	struct ipoib_neigh_hash *htbl;
1345	struct ipoib_neigh *neigh = NULL;
1346	u32 hash_val;
1347
1348	rcu_read_lock_bh();
1349
1350	htbl = rcu_dereference_bh(ntbl->htbl);
1351
1352	if (!htbl)
1353	goto out_unlock;
1354
1355	hash_val = ipoib_addr_hash(htbl, daddr);
1356	for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);
1357	neigh != NULL;
1358	neigh = rcu_dereference_bh(neigh->hnext)) {
1359	if (memcmp(p: daddr, q: neigh->daddr, INFINIBAND_ALEN) == 0) {
1360	/* found, take one ref on behalf of the caller */
1361	if (!refcount_inc_not_zero(r: &neigh->refcnt)) {
1362	/* deleted */
1363	neigh = NULL;
1364	goto out_unlock;
1365	}
1366
1367	if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))
1368	neigh->alive = jiffies;
1369	goto out_unlock;
1370	}
1371	}
1372
1373	out_unlock:
1374	rcu_read_unlock_bh();
1375	return neigh;
1376	}
1377
1378	static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
1379	{
1380	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1381	struct ipoib_neigh_hash *htbl;
1382	unsigned long neigh_obsolete;
1383	unsigned long dt;
1384	unsigned long flags;
1385	int i;
1386	LIST_HEAD(remove_list);
1387
1388	spin_lock_irqsave(&priv->lock, flags);
1389
1390	htbl = rcu_dereference_protected(ntbl->htbl,
1391	lockdep_is_held(&priv->lock));
1392
1393	if (!htbl)
1394	goto out_unlock;
1395
1396	/* neigh is obsolete if it was idle for two GC periods */
1397	dt = 2 * arp_tbl.gc_interval;
1398	neigh_obsolete = jiffies - dt;
1399
1400	for (i = 0; i < htbl->size; i++) {
1401	struct ipoib_neigh *neigh;
1402	struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1403
1404	while ((neigh = rcu_dereference_protected(*np,
1405	lockdep_is_held(&priv->lock))) != NULL) {
1406	/* was the neigh idle for two GC periods */
1407	if (time_after(neigh_obsolete, neigh->alive)) {
1408
1409	ipoib_check_and_add_mcast_sendonly(priv, mgid: neigh->daddr + 4, remove_list: &remove_list);
1410
1411	rcu_assign_pointer(*np,
1412	rcu_dereference_protected(neigh->hnext,
1413	lockdep_is_held(&priv->lock)));
1414	/* remove from path/mc list */
1415	list_del_init(entry: &neigh->list);
1416	call_rcu(head: &neigh->rcu, func: ipoib_neigh_reclaim);
1417	} else {
1418	np = &neigh->hnext;
1419	}
1420
1421	}
1422	}
1423
1424	out_unlock:
1425	spin_unlock_irqrestore(lock: &priv->lock, flags);
1426	ipoib_mcast_remove_list(remove_list: &remove_list);
1427	}
1428
1429	static void ipoib_reap_neigh(struct work_struct *work)
1430	{
1431	struct ipoib_dev_priv *priv =
1432	container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);
1433
1434	__ipoib_reap_neigh(priv);
1435
1436	queue_delayed_work(wq: priv->wq, dwork: &priv->neigh_reap_task,
1437	delay: arp_tbl.gc_interval);
1438	}
1439
1440
1441	static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,
1442	struct net_device *dev)
1443	{
1444	struct ipoib_neigh *neigh;
1445
1446	neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);
1447	if (!neigh)
1448	return NULL;
1449
1450	neigh->dev = dev;
1451	memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));
1452	skb_queue_head_init(list: &neigh->queue);
1453	INIT_LIST_HEAD(list: &neigh->list);
1454	ipoib_cm_set(neigh, NULL);
1455	/* one ref on behalf of the caller */
1456	refcount_set(r: &neigh->refcnt, n: 1);
1457
1458	return neigh;
1459	}
1460
1461	struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,
1462	struct net_device *dev)
1463	{
1464	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1465	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1466	struct ipoib_neigh_hash *htbl;
1467	struct ipoib_neigh *neigh;
1468	u32 hash_val;
1469
1470	htbl = rcu_dereference_protected(ntbl->htbl,
1471	lockdep_is_held(&priv->lock));
1472	if (!htbl) {
1473	neigh = NULL;
1474	goto out_unlock;
1475	}
1476
1477	/* need to add a new neigh, but maybe some other thread succeeded?
1478	* recalc hash, maybe hash resize took place so we do a search
1479	*/
1480	hash_val = ipoib_addr_hash(htbl, daddr);
1481	for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],
1482	lockdep_is_held(&priv->lock));
1483	neigh != NULL;
1484	neigh = rcu_dereference_protected(neigh->hnext,
1485	lockdep_is_held(&priv->lock))) {
1486	if (memcmp(p: daddr, q: neigh->daddr, INFINIBAND_ALEN) == 0) {
1487	/* found, take one ref on behalf of the caller */
1488	if (!refcount_inc_not_zero(r: &neigh->refcnt)) {
1489	/* deleted */
1490	neigh = NULL;
1491	break;
1492	}
1493	neigh->alive = jiffies;
1494	goto out_unlock;
1495	}
1496	}
1497
1498	neigh = ipoib_neigh_ctor(daddr, dev);
1499	if (!neigh)
1500	goto out_unlock;
1501
1502	/* one ref on behalf of the hash table */
1503	refcount_inc(r: &neigh->refcnt);
1504	neigh->alive = jiffies;
1505	/* put in hash */
1506	rcu_assign_pointer(neigh->hnext,
1507	rcu_dereference_protected(htbl->buckets[hash_val],
1508	lockdep_is_held(&priv->lock)));
1509	rcu_assign_pointer(htbl->buckets[hash_val], neigh);
1510	atomic_inc(v: &ntbl->entries);
1511
1512	out_unlock:
1513
1514	return neigh;
1515	}
1516
1517	void ipoib_neigh_dtor(struct ipoib_neigh *neigh)
1518	{
1519	/* neigh reference count was dropprd to zero */
1520	struct net_device *dev = neigh->dev;
1521	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1522	struct sk_buff *skb;
1523	if (neigh->ah)
1524	ipoib_put_ah(ah: neigh->ah);
1525	while ((skb = __skb_dequeue(list: &neigh->queue))) {
1526	++dev->stats.tx_dropped;
1527	dev_kfree_skb_any(skb);
1528	}
1529	if (ipoib_cm_get(neigh))
1530	ipoib_cm_destroy_tx(tx: ipoib_cm_get(neigh));
1531	ipoib_dbg(ipoib_priv(dev),
1532	"neigh free for %06x %pI6\n",
1533	IPOIB_QPN(neigh->daddr),
1534	neigh->daddr + 4);
1535	kfree(objp: neigh);
1536	if (atomic_dec_and_test(v: &priv->ntbl.entries)) {
1537	if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))
1538	complete(&priv->ntbl.flushed);
1539	}
1540	}
1541
1542	static void ipoib_neigh_reclaim(struct rcu_head *rp)
1543	{
1544	/* Called as a result of removal from hash table */
1545	struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);
1546	/* note TX context may hold another ref */
1547	ipoib_neigh_put(neigh);
1548	}
1549
1550	void ipoib_neigh_free(struct ipoib_neigh *neigh)
1551	{
1552	struct net_device *dev = neigh->dev;
1553	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1554	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1555	struct ipoib_neigh_hash *htbl;
1556	struct ipoib_neigh __rcu **np;
1557	struct ipoib_neigh *n;
1558	u32 hash_val;
1559
1560	htbl = rcu_dereference_protected(ntbl->htbl,
1561	lockdep_is_held(&priv->lock));
1562	if (!htbl)
1563	return;
1564
1565	hash_val = ipoib_addr_hash(htbl, daddr: neigh->daddr);
1566	np = &htbl->buckets[hash_val];
1567	for (n = rcu_dereference_protected(*np,
1568	lockdep_is_held(&priv->lock));
1569	n != NULL;
1570	n = rcu_dereference_protected(*np,
1571	lockdep_is_held(&priv->lock))) {
1572	if (n == neigh) {
1573	/* found */
1574	rcu_assign_pointer(*np,
1575	rcu_dereference_protected(neigh->hnext,
1576	lockdep_is_held(&priv->lock)));
1577	/* remove from parent list */
1578	list_del_init(entry: &neigh->list);
1579	call_rcu(head: &neigh->rcu, func: ipoib_neigh_reclaim);
1580	return;
1581	} else {
1582	np = &n->hnext;
1583	}
1584	}
1585	}
1586
1587	static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
1588	{
1589	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1590	struct ipoib_neigh_hash *htbl;
1591	struct ipoib_neigh __rcu **buckets;
1592	u32 size;
1593
1594	clear_bit(nr: IPOIB_NEIGH_TBL_FLUSH, addr: &priv->flags);
1595	ntbl->htbl = NULL;
1596	htbl = kzalloc(sizeof(*htbl), GFP_KERNEL);
1597	if (!htbl)
1598	return -ENOMEM;
1599	size = roundup_pow_of_two(arp_tbl.gc_thresh3);
1600	buckets = kvcalloc(size, sizeof(*buckets), GFP_KERNEL);
1601	if (!buckets) {
1602	kfree(objp: htbl);
1603	return -ENOMEM;
1604	}
1605	htbl->size = size;
1606	htbl->mask = (size - 1);
1607	htbl->buckets = buckets;
1608	RCU_INIT_POINTER(ntbl->htbl, htbl);
1609	htbl->ntbl = ntbl;
1610	atomic_set(v: &ntbl->entries, i: 0);
1611
1612	/* start garbage collection */
1613	queue_delayed_work(wq: priv->wq, dwork: &priv->neigh_reap_task,
1614	delay: arp_tbl.gc_interval);
1615
1616	return 0;
1617	}
1618
1619	static void neigh_hash_free_rcu(struct rcu_head *head)
1620	{
1621	struct ipoib_neigh_hash *htbl = container_of(head,
1622	struct ipoib_neigh_hash,
1623	rcu);
1624	struct ipoib_neigh __rcu **buckets = htbl->buckets;
1625	struct ipoib_neigh_table *ntbl = htbl->ntbl;
1626
1627	kvfree(addr: buckets);
1628	kfree(objp: htbl);
1629	complete(&ntbl->deleted);
1630	}
1631
1632	void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)
1633	{
1634	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1635	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1636	struct ipoib_neigh_hash *htbl;
1637	unsigned long flags;
1638	int i;
1639
1640	/* remove all neigh connected to a given path or mcast */
1641	spin_lock_irqsave(&priv->lock, flags);
1642
1643	htbl = rcu_dereference_protected(ntbl->htbl,
1644	lockdep_is_held(&priv->lock));
1645
1646	if (!htbl)
1647	goto out_unlock;
1648
1649	for (i = 0; i < htbl->size; i++) {
1650	struct ipoib_neigh *neigh;
1651	struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1652
1653	while ((neigh = rcu_dereference_protected(*np,
1654	lockdep_is_held(&priv->lock))) != NULL) {
1655	/* delete neighs belong to this parent */
1656	if (!memcmp(p: gid, q: neigh->daddr + 4, size: sizeof (union ib_gid))) {
1657	rcu_assign_pointer(*np,
1658	rcu_dereference_protected(neigh->hnext,
1659	lockdep_is_held(&priv->lock)));
1660	/* remove from parent list */
1661	list_del_init(entry: &neigh->list);
1662	call_rcu(head: &neigh->rcu, func: ipoib_neigh_reclaim);
1663	} else {
1664	np = &neigh->hnext;
1665	}
1666
1667	}
1668	}
1669	out_unlock:
1670	spin_unlock_irqrestore(lock: &priv->lock, flags);
1671	}
1672
1673	static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)
1674	{
1675	struct ipoib_neigh_table *ntbl = &priv->ntbl;
1676	struct ipoib_neigh_hash *htbl;
1677	unsigned long flags;
1678	int i, wait_flushed = 0;
1679
1680	init_completion(x: &priv->ntbl.flushed);
1681	set_bit(nr: IPOIB_NEIGH_TBL_FLUSH, addr: &priv->flags);
1682
1683	spin_lock_irqsave(&priv->lock, flags);
1684
1685	htbl = rcu_dereference_protected(ntbl->htbl,
1686	lockdep_is_held(&priv->lock));
1687	if (!htbl)
1688	goto out_unlock;
1689
1690	wait_flushed = atomic_read(v: &priv->ntbl.entries);
1691	if (!wait_flushed)
1692	goto free_htbl;
1693
1694	for (i = 0; i < htbl->size; i++) {
1695	struct ipoib_neigh *neigh;
1696	struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1697
1698	while ((neigh = rcu_dereference_protected(*np,
1699	lockdep_is_held(&priv->lock))) != NULL) {
1700	rcu_assign_pointer(*np,
1701	rcu_dereference_protected(neigh->hnext,
1702	lockdep_is_held(&priv->lock)));
1703	/* remove from path/mc list */
1704	list_del_init(entry: &neigh->list);
1705	call_rcu(head: &neigh->rcu, func: ipoib_neigh_reclaim);
1706	}
1707	}
1708
1709	free_htbl:
1710	rcu_assign_pointer(ntbl->htbl, NULL);
1711	call_rcu(head: &htbl->rcu, func: neigh_hash_free_rcu);
1712
1713	out_unlock:
1714	spin_unlock_irqrestore(lock: &priv->lock, flags);
1715	if (wait_flushed)
1716	wait_for_completion(&priv->ntbl.flushed);
1717	}
1718
1719	static void ipoib_neigh_hash_uninit(struct net_device *dev)
1720	{
1721	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1722
1723	ipoib_dbg(priv, "%s\n", __func__);
1724	init_completion(x: &priv->ntbl.deleted);
1725
1726	cancel_delayed_work_sync(dwork: &priv->neigh_reap_task);
1727
1728	ipoib_flush_neighs(priv);
1729
1730	wait_for_completion(&priv->ntbl.deleted);
1731	}
1732
1733	static void ipoib_napi_add(struct net_device *dev)
1734	{
1735	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1736
1737	netif_napi_add_weight(dev, napi: &priv->recv_napi, poll: ipoib_rx_poll,
1738	weight: IPOIB_NUM_WC);
1739	netif_napi_add_weight(dev, napi: &priv->send_napi, poll: ipoib_tx_poll,
1740	weight: MAX_SEND_CQE);
1741	}
1742
1743	static void ipoib_napi_del(struct net_device *dev)
1744	{
1745	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1746
1747	netif_napi_del(napi: &priv->recv_napi);
1748	netif_napi_del(napi: &priv->send_napi);
1749	}
1750
1751	static void ipoib_dev_uninit_default(struct net_device *dev)
1752	{
1753	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1754
1755	ipoib_transport_dev_cleanup(dev);
1756
1757	ipoib_napi_del(dev);
1758
1759	ipoib_cm_dev_cleanup(dev);
1760
1761	kfree(objp: priv->rx_ring);
1762	vfree(addr: priv->tx_ring);
1763
1764	priv->rx_ring = NULL;
1765	priv->tx_ring = NULL;
1766	}
1767
1768	static int ipoib_dev_init_default(struct net_device *dev)
1769	{
1770	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1771	u8 addr_mod[3];
1772
1773	ipoib_napi_add(dev);
1774
1775	/* Allocate RX/TX "rings" to hold queued skbs */
1776	priv->rx_ring = kcalloc(ipoib_recvq_size,
1777	sizeof(*priv->rx_ring),
1778	GFP_KERNEL);
1779	if (!priv->rx_ring)
1780	goto out;
1781
1782	priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,
1783	sizeof(*priv->tx_ring)));
1784	if (!priv->tx_ring) {
1785	pr_warn("%s: failed to allocate TX ring (%d entries)\n",
1786	priv->ca->name, ipoib_sendq_size);
1787	goto out_rx_ring_cleanup;
1788	}
1789
1790	/* priv->tx_head, tx_tail and global_tx_tail/head are already 0 */
1791
1792	if (ipoib_transport_dev_init(dev, ca: priv->ca)) {
1793	pr_warn("%s: ipoib_transport_dev_init failed\n",
1794	priv->ca->name);
1795	goto out_tx_ring_cleanup;
1796	}
1797
1798	/* after qp created set dev address */
1799	addr_mod[0] = (priv->qp->qp_num >> 16) & 0xff;
1800	addr_mod[1] = (priv->qp->qp_num >> 8) & 0xff;
1801	addr_mod[2] = (priv->qp->qp_num) & 0xff;
1802	dev_addr_mod(dev: priv->dev, offset: 1, addr: addr_mod, len: sizeof(addr_mod));
1803
1804	return 0;
1805
1806	out_tx_ring_cleanup:
1807	vfree(addr: priv->tx_ring);
1808
1809	out_rx_ring_cleanup:
1810	kfree(objp: priv->rx_ring);
1811
1812	out:
1813	ipoib_napi_del(dev);
1814	return -ENOMEM;
1815	}
1816
1817	static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
1818	int cmd)
1819	{
1820	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1821
1822	if (!priv->rn_ops->ndo_eth_ioctl)
1823	return -EOPNOTSUPP;
1824
1825	return priv->rn_ops->ndo_eth_ioctl(dev, ifr, cmd);
1826	}
1827
1828	static int ipoib_hwtstamp_get(struct net_device *dev,
1829	struct kernel_hwtstamp_config *config)
1830	{
1831	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1832
1833	if (!priv->rn_ops->ndo_hwtstamp_get)
1834	/* legacy */
1835	return dev_eth_ioctl(dev, ifr: config->ifr, SIOCGHWTSTAMP);
1836
1837	return priv->rn_ops->ndo_hwtstamp_get(dev, config);
1838	}
1839
1840	static int ipoib_hwtstamp_set(struct net_device *dev,
1841	struct kernel_hwtstamp_config *config,
1842	struct netlink_ext_ack *extack)
1843	{
1844	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1845
1846	if (!priv->rn_ops->ndo_hwtstamp_set)
1847	/* legacy */
1848	return dev_eth_ioctl(dev, ifr: config->ifr, SIOCSHWTSTAMP);
1849
1850	return priv->rn_ops->ndo_hwtstamp_set(dev, config, extack);
1851	}
1852
1853	static int ipoib_dev_init(struct net_device *dev)
1854	{
1855	struct ipoib_dev_priv *priv = ipoib_priv(dev);
1856	int ret = -ENOMEM;
1857
1858	priv->qp = NULL;
1859
1860	/*
1861	* the various IPoIB tasks assume they will never race against
1862	* themselves, so always use a single thread workqueue
1863	*/
1864	priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
1865	if (!priv->wq) {
1866	pr_warn("%s: failed to allocate device WQ\n", dev->name);
1867	goto out;
1868	}
1869
1870	/* create pd, which used both for control and datapath*/
1871	priv->pd = ib_alloc_pd(priv->ca, 0);
1872	if (IS_ERR(ptr: priv->pd)) {
1873	pr_warn("%s: failed to allocate PD\n", priv->ca->name);
1874	goto clean_wq;
1875	}
1876
1877	ret = priv->rn_ops->ndo_init(dev);
1878	if (ret) {
1879	pr_warn("%s failed to init HW resource\n", dev->name);
1880	goto out_free_pd;
1881	}
1882
1883	ret = ipoib_neigh_hash_init(priv);
1884	if (ret) {
1885	pr_warn("%s failed to init neigh hash\n", dev->name);
1886	goto out_dev_uninit;
1887	}
1888
1889	if (dev->flags & IFF_UP) {
1890	if (ipoib_ib_dev_open(dev)) {
1891	pr_warn("%s failed to open device\n", dev->name);
1892	ret = -ENODEV;
1893	goto out_hash_uninit;
1894	}
1895	}
1896
1897	return 0;
1898
1899	out_hash_uninit:
1900	ipoib_neigh_hash_uninit(dev);
1901
1902	out_dev_uninit:
1903	ipoib_ib_dev_cleanup(dev);
1904
1905	out_free_pd:
1906	if (priv->pd) {
1907	ib_dealloc_pd(pd: priv->pd);
1908	priv->pd = NULL;
1909	}
1910
1911	clean_wq:
1912	if (priv->wq) {
1913	destroy_workqueue(wq: priv->wq);
1914	priv->wq = NULL;
1915	}
1916
1917	out:
1918	return ret;
1919	}
1920
1921	/*
1922	* This must be called before doing an unregister_netdev on a parent device to
1923	* shutdown the IB event handler.
1924	*/
1925	static void ipoib_parent_unregister_pre(struct net_device *ndev)
1926	{
1927	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
1928
1929	/*
1930	* ipoib_set_mac checks netif_running before pushing work, clearing
1931	* running ensures the it will not add more work.
1932	*/
1933	rtnl_lock();
1934	dev_change_flags(dev: priv->dev, flags: priv->dev->flags & ~IFF_UP, NULL);
1935	rtnl_unlock();
1936
1937	/* ipoib_event() cannot be running once this returns */
1938	ib_unregister_event_handler(event_handler: &priv->event_handler);
1939
1940	/*
1941	* Work on the queue grabs the rtnl lock, so this cannot be done while
1942	* also holding it.
1943	*/
1944	flush_workqueue(ipoib_workqueue);
1945	}
1946
1947	static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)
1948	{
1949	priv->hca_caps = priv->ca->attrs.device_cap_flags;
1950	priv->kernel_caps = priv->ca->attrs.kernel_cap_flags;
1951
1952	if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
1953	priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
1954
1955	if (priv->kernel_caps & IBK_UD_TSO)
1956	priv->dev->hw_features |= NETIF_F_TSO;
1957
1958	priv->dev->features |= priv->dev->hw_features;
1959	}
1960	}
1961
1962	static int ipoib_parent_init(struct net_device *ndev)
1963	{
1964	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
1965	struct ib_port_attr attr;
1966	int result;
1967
1968	result = ib_query_port(device: priv->ca, port_num: priv->port, port_attr: &attr);
1969	if (result) {
1970	pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,
1971	priv->port);
1972	return result;
1973	}
1974	priv->max_ib_mtu = rdma_mtu_from_attr(device: priv->ca, port: priv->port, attr: &attr);
1975
1976	result = ib_query_pkey(device: priv->ca, port_num: priv->port, index: 0, pkey: &priv->pkey);
1977	if (result) {
1978	pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
1979	priv->ca->name, priv->port, result);
1980	return result;
1981	}
1982
1983	result = rdma_query_gid(device: priv->ca, port_num: priv->port, index: 0, gid: &priv->local_gid);
1984	if (result) {
1985	pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",
1986	priv->ca->name, priv->port, result);
1987	return result;
1988	}
1989	dev_addr_mod(dev: priv->dev, offset: 4, addr: priv->local_gid.raw, len: sizeof(union ib_gid));
1990
1991	SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);
1992	priv->dev->dev_port = priv->port - 1;
1993	/* Let's set this one too for backwards compatibility. */
1994	priv->dev->dev_id = priv->port - 1;
1995
1996	return 0;
1997	}
1998
1999	static void ipoib_child_init(struct net_device *ndev)
2000	{
2001	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
2002	struct ipoib_dev_priv *ppriv = ipoib_priv(dev: priv->parent);
2003
2004	priv->max_ib_mtu = ppriv->max_ib_mtu;
2005	set_bit(nr: IPOIB_FLAG_SUBINTERFACE, addr: &priv->flags);
2006	if (memchr_inv(p: priv->dev->dev_addr, c: 0, INFINIBAND_ALEN))
2007	memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
2008	sizeof(priv->local_gid));
2009	else {
2010	__dev_addr_set(dev: priv->dev, addr: ppriv->dev->dev_addr,
2011	INFINIBAND_ALEN);
2012	memcpy(&priv->local_gid, &ppriv->local_gid,
2013	sizeof(priv->local_gid));
2014	}
2015	}
2016
2017	static int ipoib_ndo_init(struct net_device *ndev)
2018	{
2019	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
2020	int rc;
2021	struct rdma_netdev *rn = netdev_priv(dev: ndev);
2022
2023	if (priv->parent) {
2024	ipoib_child_init(ndev);
2025	} else {
2026	rc = ipoib_parent_init(ndev);
2027	if (rc)
2028	return rc;
2029	}
2030
2031	/* MTU will be reset when mcast join happens */
2032	ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
2033	priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
2034	rn->mtu = priv->mcast_mtu;
2035	ndev->max_mtu = IPOIB_CM_MTU;
2036
2037	ndev->neigh_priv_len = sizeof(struct ipoib_neigh);
2038
2039	/*
2040	* Set the full membership bit, so that we join the right
2041	* broadcast group, etc.
2042	*/
2043	priv->pkey |= 0x8000;
2044
2045	ndev->broadcast[8] = priv->pkey >> 8;
2046	ndev->broadcast[9] = priv->pkey & 0xff;
2047	set_bit(nr: IPOIB_FLAG_DEV_ADDR_SET, addr: &priv->flags);
2048
2049	ipoib_set_dev_features(priv);
2050
2051	rc = ipoib_dev_init(dev: ndev);
2052	if (rc) {
2053	pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",
2054	priv->ca->name, priv->dev->name, priv->port, rc);
2055	return rc;
2056	}
2057
2058	if (priv->parent) {
2059	struct ipoib_dev_priv *ppriv = ipoib_priv(dev: priv->parent);
2060
2061	dev_hold(dev: priv->parent);
2062
2063	netdev_lock(dev: priv->parent);
2064	list_add_tail(new: &priv->list, head: &ppriv->child_intfs);
2065	netdev_unlock(dev: priv->parent);
2066	}
2067
2068	return 0;
2069	}
2070
2071	static void ipoib_ndo_uninit(struct net_device *dev)
2072	{
2073	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2074
2075	/*
2076	* ipoib_remove_one guarantees the children are removed before the
2077	* parent, and that is the only place where a parent can be removed.
2078	*/
2079	WARN_ON(!list_empty(&priv->child_intfs));
2080
2081	if (priv->parent) {
2082	struct ipoib_dev_priv *ppriv = ipoib_priv(dev: priv->parent);
2083
2084	netdev_lock(dev: ppriv->dev);
2085	list_del(entry: &priv->list);
2086	netdev_unlock(dev: ppriv->dev);
2087	}
2088
2089	ipoib_neigh_hash_uninit(dev);
2090
2091	ipoib_ib_dev_cleanup(dev);
2092
2093	/* no more works over the priv->wq */
2094	if (priv->wq) {
2095	/* See ipoib_mcast_carrier_on_task() */
2096	WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
2097	destroy_workqueue(wq: priv->wq);
2098	priv->wq = NULL;
2099	}
2100
2101	dev_put(dev: priv->parent);
2102	}
2103
2104	static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)
2105	{
2106	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2107
2108	return ib_set_vf_link_state(device: priv->ca, vf, port: priv->port, state: link_state);
2109	}
2110
2111	static int ipoib_get_vf_config(struct net_device *dev, int vf,
2112	struct ifla_vf_info *ivf)
2113	{
2114	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2115	int err;
2116
2117	err = ib_get_vf_config(device: priv->ca, vf, port: priv->port, info: ivf);
2118	if (err)
2119	return err;
2120
2121	ivf->vf = vf;
2122	memcpy(ivf->mac, dev->dev_addr, dev->addr_len);
2123
2124	return 0;
2125	}
2126
2127	static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)
2128	{
2129	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2130
2131	if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)
2132	return -EINVAL;
2133
2134	return ib_set_vf_guid(device: priv->ca, vf, port: priv->port, guid, type);
2135	}
2136
2137	static int ipoib_get_vf_guid(struct net_device *dev, int vf,
2138	struct ifla_vf_guid *node_guid,
2139	struct ifla_vf_guid *port_guid)
2140	{
2141	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2142
2143	return ib_get_vf_guid(device: priv->ca, vf, port: priv->port, node_guid, port_guid);
2144	}
2145
2146	static int ipoib_get_vf_stats(struct net_device *dev, int vf,
2147	struct ifla_vf_stats *vf_stats)
2148	{
2149	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2150
2151	return ib_get_vf_stats(device: priv->ca, vf, port: priv->port, stats: vf_stats);
2152	}
2153
2154	static const struct header_ops ipoib_header_ops = {
2155	.create = ipoib_hard_header,
2156	};
2157
2158	static const struct net_device_ops ipoib_netdev_ops_pf = {
2159	.ndo_init = ipoib_ndo_init,
2160	.ndo_uninit = ipoib_ndo_uninit,
2161	.ndo_open = ipoib_open,
2162	.ndo_stop = ipoib_stop,
2163	.ndo_change_mtu = ipoib_change_mtu,
2164	.ndo_fix_features = ipoib_fix_features,
2165	.ndo_start_xmit = ipoib_start_xmit,
2166	.ndo_tx_timeout = ipoib_timeout,
2167	.ndo_set_rx_mode = ipoib_set_mcast_list,
2168	.ndo_get_iflink = ipoib_get_iflink,
2169	.ndo_set_vf_link_state = ipoib_set_vf_link_state,
2170	.ndo_get_vf_config = ipoib_get_vf_config,
2171	.ndo_get_vf_stats = ipoib_get_vf_stats,
2172	.ndo_get_vf_guid = ipoib_get_vf_guid,
2173	.ndo_set_vf_guid = ipoib_set_vf_guid,
2174	.ndo_set_mac_address = ipoib_set_mac,
2175	.ndo_get_stats64 = ipoib_get_stats,
2176	.ndo_eth_ioctl = ipoib_ioctl,
2177	.ndo_hwtstamp_get = ipoib_hwtstamp_get,
2178	.ndo_hwtstamp_set = ipoib_hwtstamp_set,
2179	};
2180
2181	static const struct net_device_ops ipoib_netdev_ops_vf = {
2182	.ndo_init = ipoib_ndo_init,
2183	.ndo_uninit = ipoib_ndo_uninit,
2184	.ndo_open = ipoib_open,
2185	.ndo_stop = ipoib_stop,
2186	.ndo_change_mtu = ipoib_change_mtu,
2187	.ndo_fix_features = ipoib_fix_features,
2188	.ndo_start_xmit = ipoib_start_xmit,
2189	.ndo_tx_timeout = ipoib_timeout,
2190	.ndo_set_rx_mode = ipoib_set_mcast_list,
2191	.ndo_get_iflink = ipoib_get_iflink,
2192	.ndo_get_stats64 = ipoib_get_stats,
2193	.ndo_eth_ioctl = ipoib_ioctl,
2194	.ndo_hwtstamp_get = ipoib_hwtstamp_get,
2195	.ndo_hwtstamp_set = ipoib_hwtstamp_set,
2196	};
2197
2198	static const struct net_device_ops ipoib_netdev_default_pf = {
2199	.ndo_init = ipoib_dev_init_default,
2200	.ndo_uninit = ipoib_dev_uninit_default,
2201	.ndo_open = ipoib_ib_dev_open_default,
2202	.ndo_stop = ipoib_ib_dev_stop_default,
2203	};
2204
2205	void ipoib_setup_common(struct net_device *dev)
2206	{
2207	dev->header_ops = &ipoib_header_ops;
2208	dev->netdev_ops = &ipoib_netdev_default_pf;
2209
2210	ipoib_set_ethtool_ops(dev);
2211
2212	dev->watchdog_timeo = 10 * HZ;
2213
2214	dev->flags |= IFF_BROADCAST | IFF_MULTICAST;
2215
2216	dev->hard_header_len = IPOIB_HARD_LEN;
2217	dev->addr_len = INFINIBAND_ALEN;
2218	dev->type = ARPHRD_INFINIBAND;
2219	dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
2220	dev->features = (NETIF_F_VLAN_CHALLENGED |
2221	NETIF_F_HIGHDMA);
2222	netif_keep_dst(dev);
2223
2224	memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);
2225
2226	/*
2227	* unregister_netdev always frees the netdev, we use this mode
2228	* consistently to unify all the various unregister paths, including
2229	* those connected to rtnl_link_ops which require it.
2230	*/
2231	dev->needs_free_netdev = true;
2232	}
2233
2234	static void ipoib_build_priv(struct net_device *dev)
2235	{
2236	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2237
2238	priv->dev = dev;
2239	spin_lock_init(&priv->lock);
2240	mutex_init(&priv->mcast_mutex);
2241
2242	INIT_LIST_HEAD(list: &priv->path_list);
2243	INIT_LIST_HEAD(list: &priv->child_intfs);
2244	INIT_LIST_HEAD(list: &priv->dead_ahs);
2245	INIT_LIST_HEAD(list: &priv->multicast_list);
2246
2247	INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task);
2248	INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);
2249	INIT_WORK(&priv->reschedule_napi_work, ipoib_napi_schedule_work);
2250	INIT_WORK(&priv->flush_light, ipoib_ib_dev_flush_light);
2251	INIT_WORK(&priv->flush_normal, ipoib_ib_dev_flush_normal);
2252	INIT_WORK(&priv->flush_heavy, ipoib_ib_dev_flush_heavy);
2253	INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
2254	INIT_WORK(&priv->tx_timeout_work, ipoib_ib_tx_timeout_work);
2255	INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
2256	INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
2257	}
2258
2259	static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u32 port,
2260	const char *name)
2261	{
2262	struct net_device *dev;
2263
2264	dev = rdma_alloc_netdev(device: hca, port_num: port, type: RDMA_NETDEV_IPOIB, name,
2265	NET_NAME_UNKNOWN, setup: ipoib_setup_common);
2266	if (!IS_ERR(ptr: dev) || PTR_ERR(ptr: dev) != -EOPNOTSUPP)
2267	return dev;
2268
2269	dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,
2270	ipoib_setup_common);
2271	if (!dev)
2272	return ERR_PTR(error: -ENOMEM);
2273	return dev;
2274	}
2275
2276	int ipoib_intf_init(struct ib_device *hca, u32 port, const char *name,
2277	struct net_device *dev)
2278	{
2279	struct rdma_netdev *rn = netdev_priv(dev);
2280	struct ipoib_dev_priv *priv;
2281	int rc;
2282
2283	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2284	if (!priv)
2285	return -ENOMEM;
2286
2287	priv->ca = hca;
2288	priv->port = port;
2289
2290	rc = rdma_init_netdev(device: hca, port_num: port, type: RDMA_NETDEV_IPOIB, name,
2291	NET_NAME_UNKNOWN, setup: ipoib_setup_common, netdev: dev);
2292	if (rc) {
2293	if (rc != -EOPNOTSUPP)
2294	goto out;
2295
2296	rn->send = ipoib_send;
2297	rn->attach_mcast = ipoib_mcast_attach;
2298	rn->detach_mcast = ipoib_mcast_detach;
2299	rn->hca = hca;
2300
2301	rc = netif_set_real_num_tx_queues(dev, txq: 1);
2302	if (rc)
2303	goto out;
2304
2305	rc = netif_set_real_num_rx_queues(dev, rxq: 1);
2306	if (rc)
2307	goto out;
2308	}
2309
2310	priv->rn_ops = dev->netdev_ops;
2311
2312	if (hca->attrs.kernel_cap_flags & IBK_VIRTUAL_FUNCTION)
2313	dev->netdev_ops = &ipoib_netdev_ops_vf;
2314	else
2315	dev->netdev_ops = &ipoib_netdev_ops_pf;
2316
2317	rn->clnt_priv = priv;
2318	/*
2319	* Only the child register_netdev flows can handle priv_destructor
2320	* being set, so we force it to NULL here and handle manually until it
2321	* is safe to turn on.
2322	*/
2323	priv->next_priv_destructor = dev->priv_destructor;
2324	dev->priv_destructor = NULL;
2325
2326	ipoib_build_priv(dev);
2327
2328	return 0;
2329
2330	out:
2331	kfree(objp: priv);
2332	return rc;
2333	}
2334
2335	struct net_device *ipoib_intf_alloc(struct ib_device *hca, u32 port,
2336	const char *name)
2337	{
2338	struct net_device *dev;
2339	int rc;
2340
2341	dev = ipoib_alloc_netdev(hca, port, name);
2342	if (IS_ERR(ptr: dev))
2343	return dev;
2344
2345	rc = ipoib_intf_init(hca, port, name, dev);
2346	if (rc) {
2347	free_netdev(dev);
2348	return ERR_PTR(error: rc);
2349	}
2350
2351	/*
2352	* Upon success the caller must ensure ipoib_intf_free is called or
2353	* register_netdevice succeed'd and priv_destructor is set to
2354	* ipoib_intf_free.
2355	*/
2356	return dev;
2357	}
2358
2359	void ipoib_intf_free(struct net_device *dev)
2360	{
2361	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2362	struct rdma_netdev *rn = netdev_priv(dev);
2363
2364	dev->priv_destructor = priv->next_priv_destructor;
2365	if (dev->priv_destructor)
2366	dev->priv_destructor(dev);
2367
2368	/*
2369	* There are some error flows around register_netdev failing that may
2370	* attempt to call priv_destructor twice, prevent that from happening.
2371	*/
2372	dev->priv_destructor = NULL;
2373
2374	/* unregister/destroy is very complicated. Make bugs more obvious. */
2375	rn->clnt_priv = NULL;
2376
2377	kfree(objp: priv);
2378	}
2379
2380	static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,
2381	char *buf)
2382	{
2383	struct net_device *ndev = to_net_dev(dev);
2384	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
2385
2386	return sysfs_emit(buf, fmt: "0x%04x\n", priv->pkey);
2387	}
2388	static DEVICE_ATTR_RO(pkey);
2389
2390	static ssize_t umcast_show(struct device *dev, struct device_attribute *attr,
2391	char *buf)
2392	{
2393	struct net_device *ndev = to_net_dev(dev);
2394	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
2395
2396	return sysfs_emit(buf, fmt: "%d\n",
2397	test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
2398	}
2399
2400	void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
2401	{
2402	struct ipoib_dev_priv *priv = ipoib_priv(dev: ndev);
2403
2404	if (umcast_val > 0) {
2405	set_bit(nr: IPOIB_FLAG_UMCAST, addr: &priv->flags);
2406	ipoib_warn(priv, "ignoring multicast groups joined directly "
2407	"by userspace\n");
2408	} else
2409	clear_bit(nr: IPOIB_FLAG_UMCAST, addr: &priv->flags);
2410	}
2411
2412	static ssize_t umcast_store(struct device *dev, struct device_attribute *attr,
2413	const char *buf, size_t count)
2414	{
2415	unsigned long umcast_val = simple_strtoul(buf, NULL, 0);
2416
2417	ipoib_set_umcast(to_net_dev(dev), umcast_val);
2418
2419	return count;
2420	}
2421	static DEVICE_ATTR_RW(umcast);
2422
2423	int ipoib_add_umcast_attr(struct net_device *dev)
2424	{
2425	return device_create_file(device: &dev->dev, entry: &dev_attr_umcast);
2426	}
2427
2428	static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)
2429	{
2430	struct ipoib_dev_priv *child_priv;
2431	struct net_device *netdev = priv->dev;
2432
2433	netif_addr_lock_bh(dev: netdev);
2434
2435	memcpy(&priv->local_gid.global.interface_id,
2436	&gid->global.interface_id,
2437	sizeof(gid->global.interface_id));
2438	dev_addr_mod(dev: netdev, offset: 4, addr: (u8 *)&priv->local_gid, len: sizeof(priv->local_gid));
2439	clear_bit(nr: IPOIB_FLAG_DEV_ADDR_SET, addr: &priv->flags);
2440
2441	netif_addr_unlock_bh(dev: netdev);
2442
2443	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
2444	netdev_lock_ops_to_full(dev: priv->dev);
2445	list_for_each_entry(child_priv, &priv->child_intfs, list)
2446	set_base_guid(priv: child_priv, gid);
2447	netdev_unlock_full_to_ops(dev: priv->dev);
2448	}
2449	}
2450
2451	static int ipoib_check_lladdr(struct net_device *dev,
2452	struct sockaddr_storage *ss)
2453	{
2454	union ib_gid *gid = (union ib_gid *)(ss->__data + 4);
2455	int ret = 0;
2456
2457	netif_addr_lock_bh(dev);
2458
2459	/* Make sure the QPN, reserved and subnet prefix match the current
2460	* lladdr, it also makes sure the lladdr is unicast.
2461	*/
2462	if (memcmp(p: dev->dev_addr, q: ss->__data,
2463	size: 4 + sizeof(gid->global.subnet_prefix)) ||
2464	gid->global.interface_id == 0)
2465	ret = -EINVAL;
2466
2467	netif_addr_unlock_bh(dev);
2468
2469	return ret;
2470	}
2471
2472	static int ipoib_set_mac(struct net_device *dev, void *addr)
2473	{
2474	struct ipoib_dev_priv *priv = ipoib_priv(dev);
2475	struct sockaddr_storage *ss = addr;
2476	int ret;
2477
2478	if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
2479	return -EBUSY;
2480
2481	ret = ipoib_check_lladdr(dev, ss);
2482	if (ret)
2483	return ret;
2484
2485	set_base_guid(priv, gid: (union ib_gid *)(ss->__data + 4));
2486
2487	if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
2488	struct ipoib_dev_priv *cpriv;
2489
2490	netdev_lock_ops_to_full(dev);
2491	list_for_each_entry(cpriv, &priv->child_intfs, list)
2492	queue_work(wq: ipoib_workqueue, work: &cpriv->flush_light);
2493	netdev_unlock_full_to_ops(dev);
2494	}
2495	queue_work(wq: ipoib_workqueue, work: &priv->flush_light);
2496
2497	return 0;
2498	}
2499
2500	static ssize_t create_child_store(struct device *dev,
2501	struct device_attribute *attr,
2502	const char *buf, size_t count)
2503	{
2504	int pkey;
2505	int ret;
2506
2507	if (sscanf(buf, "%i", &pkey) != 1)
2508	return -EINVAL;
2509
2510	if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)
2511	return -EINVAL;
2512
2513	ret = ipoib_vlan_add(to_net_dev(dev), pkey);
2514
2515	return ret ? ret : count;
2516	}
2517	static DEVICE_ATTR_WO(create_child);
2518
2519	static ssize_t delete_child_store(struct device *dev,
2520	struct device_attribute *attr,
2521	const char *buf, size_t count)
2522	{
2523	int pkey;
2524	int ret;
2525
2526	if (sscanf(buf, "%i", &pkey) != 1)
2527	return -EINVAL;
2528
2529	if (pkey < 0 || pkey > 0xffff)
2530	return -EINVAL;
2531
2532	ret = ipoib_vlan_delete(to_net_dev(dev), pkey);
2533
2534	return ret ? ret : count;
2535
2536	}
2537	static DEVICE_ATTR_WO(delete_child);
2538
2539	int ipoib_add_pkey_attr(struct net_device *dev)
2540	{
2541	return device_create_file(device: &dev->dev, entry: &dev_attr_pkey);
2542	}
2543
2544	/*
2545	* We erroneously exposed the iface's port number in the dev_id
2546	* sysfs field long after dev_port was introduced for that purpose[1],
2547	* and we need to stop everyone from relying on that.
2548	* Let's overload the shower routine for the dev_id file here
2549	* to gently bring the issue up.
2550	*
2551	* [1] https://www.spinics.net/lists/netdev/msg272123.html
2552	*/
2553	static ssize_t dev_id_show(struct device *dev,
2554	struct device_attribute *attr, char *buf)
2555	{
2556	struct net_device *ndev = to_net_dev(dev);
2557
2558	/*
2559	* ndev->dev_port will be equal to 0 in old kernel prior to commit
2560	* 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
2561	* port numbers") Zero was chosen as special case for user space
2562	* applications to fallback and query dev_id to check if it has
2563	* different value or not.
2564	*
2565	* Don't print warning in such scenario.
2566	*
2567	* https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
2568	*/
2569	if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
2570	netdev_info_once(ndev,
2571	"\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
2572	current->comm);
2573
2574	return sysfs_emit(buf, fmt: "%#x\n", ndev->dev_id);
2575	}
2576	static DEVICE_ATTR_RO(dev_id);
2577
2578	static int ipoib_intercept_dev_id_attr(struct net_device *dev)
2579	{
2580	device_remove_file(dev: &dev->dev, attr: &dev_attr_dev_id);
2581	return device_create_file(device: &dev->dev, entry: &dev_attr_dev_id);
2582	}
2583
2584	static struct net_device *ipoib_add_port(const char *format,
2585	struct ib_device *hca, u32 port)
2586	{
2587	struct rtnl_link_ops *ops = ipoib_get_link_ops();
2588	struct rdma_netdev_alloc_params params;
2589	struct ipoib_dev_priv *priv;
2590	struct net_device *ndev;
2591	int result;
2592
2593	ndev = ipoib_intf_alloc(hca, port, name: format);
2594	if (IS_ERR(ptr: ndev)) {
2595	pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,
2596	PTR_ERR(ndev));
2597	return ndev;
2598	}
2599	priv = ipoib_priv(dev: ndev);
2600
2601	INIT_IB_EVENT_HANDLER(&priv->event_handler,
2602	priv->ca, ipoib_event);
2603	ib_register_event_handler(event_handler: &priv->event_handler);
2604
2605	/* call event handler to ensure pkey in sync */
2606	ipoib_queue_work(priv, level: IPOIB_FLUSH_HEAVY);
2607
2608	ndev->rtnl_link_ops = ipoib_get_link_ops();
2609
2610	dev_net_set(dev: ndev, net: rdma_dev_net(device: hca));
2611
2612	result = register_netdev(dev: ndev);
2613	if (result) {
2614	pr_warn("%s: couldn't register ipoib port %d; error %d\n",
2615	hca->name, port, result);
2616
2617	ipoib_parent_unregister_pre(ndev);
2618	ipoib_intf_free(dev: ndev);
2619	free_netdev(dev: ndev);
2620
2621	return ERR_PTR(error: result);
2622	}
2623
2624	if (hca->ops.rdma_netdev_get_params) {
2625	int rc = hca->ops.rdma_netdev_get_params(hca, port,
2626	RDMA_NETDEV_IPOIB,
2627	&params);
2628
2629	if (!rc && ops->priv_size < params.sizeof_priv)
2630	ops->priv_size = params.sizeof_priv;
2631	}
2632	/*
2633	* We cannot set priv_destructor before register_netdev because we
2634	* need priv to be always valid during the error flow to execute
2635	* ipoib_parent_unregister_pre(). Instead handle it manually and only
2636	* enter priv_destructor mode once we are completely registered.
2637	*/
2638	ndev->priv_destructor = ipoib_intf_free;
2639
2640	if (ipoib_intercept_dev_id_attr(dev: ndev))
2641	goto sysfs_failed;
2642	if (ipoib_cm_add_mode_attr(dev: ndev))
2643	goto sysfs_failed;
2644	if (ipoib_add_pkey_attr(dev: ndev))
2645	goto sysfs_failed;
2646	if (ipoib_add_umcast_attr(dev: ndev))
2647	goto sysfs_failed;
2648	if (device_create_file(device: &ndev->dev, entry: &dev_attr_create_child))
2649	goto sysfs_failed;
2650	if (device_create_file(device: &ndev->dev, entry: &dev_attr_delete_child))
2651	goto sysfs_failed;
2652
2653	return ndev;
2654
2655	sysfs_failed:
2656	ipoib_parent_unregister_pre(ndev);
2657	unregister_netdev(dev: ndev);
2658	return ERR_PTR(error: -ENOMEM);
2659	}
2660
2661	static int ipoib_add_one(struct ib_device *device)
2662	{
2663	struct list_head *dev_list;
2664	struct net_device *dev;
2665	struct ipoib_dev_priv *priv;
2666	unsigned int p;
2667	int count = 0;
2668
2669	dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
2670	if (!dev_list)
2671	return -ENOMEM;
2672
2673	INIT_LIST_HEAD(list: dev_list);
2674
2675	rdma_for_each_port (device, p) {
2676	if (!rdma_protocol_ib(device, port_num: p))
2677	continue;
2678	dev = ipoib_add_port(format: "ib%d", hca: device, port: p);
2679	if (!IS_ERR(ptr: dev)) {
2680	priv = ipoib_priv(dev);
2681	list_add_tail(new: &priv->list, head: dev_list);
2682	count++;
2683	}
2684	}
2685
2686	if (!count) {
2687	kfree(objp: dev_list);
2688	return -EOPNOTSUPP;
2689	}
2690
2691	ib_set_client_data(device, client: &ipoib_client, data: dev_list);
2692	return 0;
2693	}
2694
2695	static void ipoib_remove_one(struct ib_device *device, void *client_data)
2696	{
2697	struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
2698	struct list_head *dev_list = client_data;
2699
2700	list_for_each_entry_safe(priv, tmp, dev_list, list) {
2701	LIST_HEAD(head);
2702	ipoib_parent_unregister_pre(ndev: priv->dev);
2703
2704	rtnl_lock();
2705
2706	netdev_lock(dev: priv->dev);
2707	list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,
2708	list)
2709	unregister_netdevice_queue(dev: cpriv->dev, head: &head);
2710	netdev_unlock(dev: priv->dev);
2711	unregister_netdevice_queue(dev: priv->dev, head: &head);
2712	unregister_netdevice_many(head: &head);
2713
2714	rtnl_unlock();
2715	}
2716
2717	kfree(objp: dev_list);
2718	}
2719
2720	#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2721	static struct notifier_block ipoib_netdev_notifier = {
2722	.notifier_call = ipoib_netdev_event,
2723	};
2724	#endif
2725
2726	static int __init ipoib_init_module(void)
2727	{
2728	int ret;
2729
2730	ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);
2731	ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);
2732	ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);
2733
2734	ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);
2735	ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);
2736	ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);
2737	#ifdef CONFIG_INFINIBAND_IPOIB_CM
2738	ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
2739	ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);
2740	#endif
2741
2742	/*
2743	* When copying small received packets, we only copy from the
2744	* linear data part of the SKB, so we rely on this condition.
2745	*/
2746	BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);
2747
2748	ipoib_register_debugfs();
2749
2750	/*
2751	* We create a global workqueue here that is used for all flush
2752	* operations. However, if you attempt to flush a workqueue
2753	* from a task on that same workqueue, it deadlocks the system.
2754	* We want to be able to flush the tasks associated with a
2755	* specific net device, so we also create a workqueue for each
2756	* netdevice. We queue up the tasks for that device only on
2757	* its private workqueue, and we only queue up flush events
2758	* on our global flush workqueue. This avoids the deadlocks.
2759	*/
2760	ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);
2761	if (!ipoib_workqueue) {
2762	ret = -ENOMEM;
2763	goto err_fs;
2764	}
2765
2766	ib_sa_register_client(client: &ipoib_sa_client);
2767
2768	ret = ib_register_client(client: &ipoib_client);
2769	if (ret)
2770	goto err_sa;
2771
2772	ret = ipoib_netlink_init();
2773	if (ret)
2774	goto err_client;
2775
2776	#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2777	register_netdevice_notifier(nb: &ipoib_netdev_notifier);
2778	#endif
2779	return 0;
2780
2781	err_client:
2782	ib_unregister_client(client: &ipoib_client);
2783
2784	err_sa:
2785	ib_sa_unregister_client(client: &ipoib_sa_client);
2786	destroy_workqueue(wq: ipoib_workqueue);
2787
2788	err_fs:
2789	ipoib_unregister_debugfs();
2790
2791	return ret;
2792	}
2793
2794	static void __exit ipoib_cleanup_module(void)
2795	{
2796	#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2797	unregister_netdevice_notifier(nb: &ipoib_netdev_notifier);
2798	#endif
2799	ipoib_netlink_fini();
2800	ib_unregister_client(client: &ipoib_client);
2801	ib_sa_unregister_client(client: &ipoib_sa_client);
2802	ipoib_unregister_debugfs();
2803	destroy_workqueue(wq: ipoib_workqueue);
2804	}
2805
2806	module_init(ipoib_init_module);
2807	module_exit(ipoib_cleanup_module);
2808