1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2022 Hewlett Packard Enterprise, Inc. All rights reserved.
4 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
5 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
6 */
7
8/*
9 * rxe_mcast.c implements driver support for multicast transport.
10 * It is based on two data structures struct rxe_mcg ('mcg') and
11 * struct rxe_mca ('mca'). An mcg is allocated each time a qp is
12 * attached to a new mgid for the first time. These are indexed by
13 * a red-black tree using the mgid. This data structure is searched
14 * for the mcg when a multicast packet is received and when another
15 * qp is attached to the same mgid. It is cleaned up when the last qp
16 * is detached from the mcg. Each time a qp is attached to an mcg an
17 * mca is created. It holds a pointer to the qp and is added to a list
18 * of qp's that are attached to the mcg. The qp_list is used to replicate
19 * mcast packets in the rxe receive path.
20 */
21
22#include "rxe.h"
23
24/**
25 * rxe_mcast_add - add multicast address to rxe device
26 * @rxe: rxe device object
27 * @mgid: multicast address as a gid
28 *
29 * Returns 0 on success else an error
30 */
31static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)
32{
33 unsigned char ll_addr[ETH_ALEN];
34 struct net_device *ndev;
35 int ret;
36
37 ndev = rxe_ib_device_get_netdev(dev: &rxe->ib_dev);
38 if (!ndev)
39 return -ENODEV;
40
41 ipv6_eth_mc_map(addr: (struct in6_addr *)mgid->raw, buf: ll_addr);
42
43 ret = dev_mc_add(dev: ndev, addr: ll_addr);
44 dev_put(dev: ndev);
45
46 return ret;
47}
48
49/**
50 * rxe_mcast_del - delete multicast address from rxe device
51 * @rxe: rxe device object
52 * @mgid: multicast address as a gid
53 *
54 * Returns 0 on success else an error
55 */
56static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid)
57{
58 unsigned char ll_addr[ETH_ALEN];
59 struct net_device *ndev;
60 int ret;
61
62 ndev = rxe_ib_device_get_netdev(dev: &rxe->ib_dev);
63 if (!ndev)
64 return -ENODEV;
65
66 ipv6_eth_mc_map(addr: (struct in6_addr *)mgid->raw, buf: ll_addr);
67
68 ret = dev_mc_del(dev: ndev, addr: ll_addr);
69 dev_put(dev: ndev);
70
71 return ret;
72}
73
74/**
75 * __rxe_insert_mcg - insert an mcg into red-black tree (rxe->mcg_tree)
76 * @mcg: mcg object with an embedded red-black tree node
77 *
78 * Context: caller must hold a reference to mcg and rxe->mcg_lock and
79 * is responsible to avoid adding the same mcg twice to the tree.
80 */
81static void __rxe_insert_mcg(struct rxe_mcg *mcg)
82{
83 struct rb_root *tree = &mcg->rxe->mcg_tree;
84 struct rb_node **link = &tree->rb_node;
85 struct rb_node *node = NULL;
86 struct rxe_mcg *tmp;
87 int cmp;
88
89 while (*link) {
90 node = *link;
91 tmp = rb_entry(node, struct rxe_mcg, node);
92
93 cmp = memcmp(p: &tmp->mgid, q: &mcg->mgid, size: sizeof(mcg->mgid));
94 if (cmp > 0)
95 link = &(*link)->rb_left;
96 else
97 link = &(*link)->rb_right;
98 }
99
100 rb_link_node(node: &mcg->node, parent: node, rb_link: link);
101 rb_insert_color(&mcg->node, tree);
102}
103
104/**
105 * __rxe_remove_mcg - remove an mcg from red-black tree holding lock
106 * @mcg: mcast group object with an embedded red-black tree node
107 *
108 * Context: caller must hold a reference to mcg and rxe->mcg_lock
109 */
110static void __rxe_remove_mcg(struct rxe_mcg *mcg)
111{
112 rb_erase(&mcg->node, &mcg->rxe->mcg_tree);
113}
114
115/**
116 * __rxe_lookup_mcg - lookup mcg in rxe->mcg_tree while holding lock
117 * @rxe: rxe device object
118 * @mgid: multicast IP address
119 *
120 * Context: caller must hold rxe->mcg_lock
121 * Returns: mcg on success and takes a ref to mcg else NULL
122 */
123static struct rxe_mcg *__rxe_lookup_mcg(struct rxe_dev *rxe,
124 union ib_gid *mgid)
125{
126 struct rb_root *tree = &rxe->mcg_tree;
127 struct rxe_mcg *mcg;
128 struct rb_node *node;
129 int cmp;
130
131 node = tree->rb_node;
132
133 while (node) {
134 mcg = rb_entry(node, struct rxe_mcg, node);
135
136 cmp = memcmp(p: &mcg->mgid, q: mgid, size: sizeof(*mgid));
137
138 if (cmp > 0)
139 node = node->rb_left;
140 else if (cmp < 0)
141 node = node->rb_right;
142 else
143 break;
144 }
145
146 if (node) {
147 kref_get(kref: &mcg->ref_cnt);
148 return mcg;
149 }
150
151 return NULL;
152}
153
154/**
155 * rxe_lookup_mcg - lookup up mcg in red-back tree
156 * @rxe: rxe device object
157 * @mgid: multicast IP address
158 *
159 * Returns: mcg if found else NULL
160 */
161struct rxe_mcg *rxe_lookup_mcg(struct rxe_dev *rxe, union ib_gid *mgid)
162{
163 struct rxe_mcg *mcg;
164
165 spin_lock_bh(lock: &rxe->mcg_lock);
166 mcg = __rxe_lookup_mcg(rxe, mgid);
167 spin_unlock_bh(lock: &rxe->mcg_lock);
168
169 return mcg;
170}
171
172/**
173 * __rxe_init_mcg - initialize a new mcg
174 * @rxe: rxe device
175 * @mgid: multicast address as a gid
176 * @mcg: new mcg object
177 *
178 * Context: caller should hold rxe->mcg lock
179 */
180static void __rxe_init_mcg(struct rxe_dev *rxe, union ib_gid *mgid,
181 struct rxe_mcg *mcg)
182{
183 kref_init(kref: &mcg->ref_cnt);
184 memcpy(&mcg->mgid, mgid, sizeof(mcg->mgid));
185 INIT_LIST_HEAD(list: &mcg->qp_list);
186 mcg->rxe = rxe;
187
188 /* caller holds a ref on mcg but that will be
189 * dropped when mcg goes out of scope. We need to take a ref
190 * on the pointer that will be saved in the red-black tree
191 * by __rxe_insert_mcg and used to lookup mcg from mgid later.
192 * Inserting mcg makes it visible to outside so this should
193 * be done last after the object is ready.
194 */
195 kref_get(kref: &mcg->ref_cnt);
196 __rxe_insert_mcg(mcg);
197}
198
199/**
200 * rxe_get_mcg - lookup or allocate a mcg
201 * @rxe: rxe device object
202 * @mgid: multicast IP address as a gid
203 *
204 * Returns: mcg on success else ERR_PTR(error)
205 */
206static struct rxe_mcg *rxe_get_mcg(struct rxe_dev *rxe, union ib_gid *mgid)
207{
208 struct rxe_mcg *mcg, *tmp;
209 int err;
210
211 if (rxe->attr.max_mcast_grp == 0)
212 return ERR_PTR(error: -EINVAL);
213
214 /* check to see if mcg already exists */
215 mcg = rxe_lookup_mcg(rxe, mgid);
216 if (mcg)
217 return mcg;
218
219 /* check to see if we have reached limit */
220 if (atomic_inc_return(v: &rxe->mcg_num) > rxe->attr.max_mcast_grp) {
221 err = -ENOMEM;
222 goto err_dec;
223 }
224
225 /* speculative alloc of new mcg */
226 mcg = kzalloc(sizeof(*mcg), GFP_KERNEL);
227 if (!mcg) {
228 err = -ENOMEM;
229 goto err_dec;
230 }
231
232 spin_lock_bh(lock: &rxe->mcg_lock);
233 /* re-check to see if someone else just added it */
234 tmp = __rxe_lookup_mcg(rxe, mgid);
235 if (tmp) {
236 spin_unlock_bh(lock: &rxe->mcg_lock);
237 atomic_dec(v: &rxe->mcg_num);
238 kfree(objp: mcg);
239 return tmp;
240 }
241
242 __rxe_init_mcg(rxe, mgid, mcg);
243 spin_unlock_bh(lock: &rxe->mcg_lock);
244
245 /* add mcast address outside of lock */
246 err = rxe_mcast_add(rxe, mgid);
247 if (!err)
248 return mcg;
249
250 kfree(objp: mcg);
251err_dec:
252 atomic_dec(v: &rxe->mcg_num);
253 return ERR_PTR(error: err);
254}
255
256/**
257 * rxe_cleanup_mcg - cleanup mcg for kref_put
258 * @kref: struct kref embnedded in mcg
259 */
260void rxe_cleanup_mcg(struct kref *kref)
261{
262 struct rxe_mcg *mcg = container_of(kref, typeof(*mcg), ref_cnt);
263
264 kfree(objp: mcg);
265}
266
267/**
268 * __rxe_destroy_mcg - destroy mcg object holding rxe->mcg_lock
269 * @mcg: the mcg object
270 *
271 * Context: caller is holding rxe->mcg_lock
272 * no qp's are attached to mcg
273 */
274static void __rxe_destroy_mcg(struct rxe_mcg *mcg)
275{
276 struct rxe_dev *rxe = mcg->rxe;
277
278 /* remove mcg from red-black tree then drop ref */
279 __rxe_remove_mcg(mcg);
280 kref_put(kref: &mcg->ref_cnt, release: rxe_cleanup_mcg);
281
282 atomic_dec(v: &rxe->mcg_num);
283}
284
285/**
286 * rxe_destroy_mcg - destroy mcg object
287 * @mcg: the mcg object
288 *
289 * Context: no qp's are attached to mcg
290 */
291static void rxe_destroy_mcg(struct rxe_mcg *mcg)
292{
293 /* delete mcast address outside of lock */
294 rxe_mcast_del(rxe: mcg->rxe, mgid: &mcg->mgid);
295
296 spin_lock_bh(lock: &mcg->rxe->mcg_lock);
297 __rxe_destroy_mcg(mcg);
298 spin_unlock_bh(lock: &mcg->rxe->mcg_lock);
299}
300
301/**
302 * __rxe_init_mca - initialize a new mca holding lock
303 * @qp: qp object
304 * @mcg: mcg object
305 * @mca: empty space for new mca
306 *
307 * Context: caller must hold references on qp and mcg, rxe->mcg_lock
308 * and pass memory for new mca
309 *
310 * Returns: 0 on success else an error
311 */
312static int __rxe_init_mca(struct rxe_qp *qp, struct rxe_mcg *mcg,
313 struct rxe_mca *mca)
314{
315 struct rxe_dev *rxe = to_rdev(dev: qp->ibqp.device);
316 int n;
317
318 n = atomic_inc_return(v: &rxe->mcg_attach);
319 if (n > rxe->attr.max_total_mcast_qp_attach) {
320 atomic_dec(v: &rxe->mcg_attach);
321 return -ENOMEM;
322 }
323
324 n = atomic_inc_return(v: &mcg->qp_num);
325 if (n > rxe->attr.max_mcast_qp_attach) {
326 atomic_dec(v: &mcg->qp_num);
327 atomic_dec(v: &rxe->mcg_attach);
328 return -ENOMEM;
329 }
330
331 atomic_inc(v: &qp->mcg_num);
332
333 rxe_get(qp);
334 mca->qp = qp;
335
336 list_add_tail(new: &mca->qp_list, head: &mcg->qp_list);
337
338 return 0;
339}
340
341/**
342 * rxe_attach_mcg - attach qp to mcg if not already attached
343 * @qp: qp object
344 * @mcg: mcg object
345 *
346 * Context: caller must hold reference on qp and mcg.
347 * Returns: 0 on success else an error
348 */
349static int rxe_attach_mcg(struct rxe_mcg *mcg, struct rxe_qp *qp)
350{
351 struct rxe_dev *rxe = mcg->rxe;
352 struct rxe_mca *mca, *tmp;
353 int err;
354
355 /* check to see if the qp is already a member of the group */
356 spin_lock_bh(lock: &rxe->mcg_lock);
357 list_for_each_entry(mca, &mcg->qp_list, qp_list) {
358 if (mca->qp == qp) {
359 spin_unlock_bh(lock: &rxe->mcg_lock);
360 return 0;
361 }
362 }
363 spin_unlock_bh(lock: &rxe->mcg_lock);
364
365 /* speculative alloc new mca without using GFP_ATOMIC */
366 mca = kzalloc(sizeof(*mca), GFP_KERNEL);
367 if (!mca)
368 return -ENOMEM;
369
370 spin_lock_bh(lock: &rxe->mcg_lock);
371 /* re-check to see if someone else just attached qp */
372 list_for_each_entry(tmp, &mcg->qp_list, qp_list) {
373 if (tmp->qp == qp) {
374 kfree(objp: mca);
375 err = 0;
376 goto out;
377 }
378 }
379
380 err = __rxe_init_mca(qp, mcg, mca);
381 if (err)
382 kfree(objp: mca);
383out:
384 spin_unlock_bh(lock: &rxe->mcg_lock);
385 return err;
386}
387
388/**
389 * __rxe_cleanup_mca - cleanup mca object holding lock
390 * @mca: mca object
391 * @mcg: mcg object
392 *
393 * Context: caller must hold a reference to mcg and rxe->mcg_lock
394 */
395static void __rxe_cleanup_mca(struct rxe_mca *mca, struct rxe_mcg *mcg)
396{
397 list_del(entry: &mca->qp_list);
398
399 atomic_dec(v: &mcg->qp_num);
400 atomic_dec(v: &mcg->rxe->mcg_attach);
401 atomic_dec(v: &mca->qp->mcg_num);
402 rxe_put(mca->qp);
403
404 kfree(objp: mca);
405}
406
407/**
408 * rxe_detach_mcg - detach qp from mcg
409 * @mcg: mcg object
410 * @qp: qp object
411 *
412 * Returns: 0 on success else an error if qp is not attached.
413 */
414static int rxe_detach_mcg(struct rxe_mcg *mcg, struct rxe_qp *qp)
415{
416 struct rxe_dev *rxe = mcg->rxe;
417 struct rxe_mca *mca, *tmp;
418
419 spin_lock_bh(lock: &rxe->mcg_lock);
420 list_for_each_entry_safe(mca, tmp, &mcg->qp_list, qp_list) {
421 if (mca->qp == qp) {
422 __rxe_cleanup_mca(mca, mcg);
423
424 /* if the number of qp's attached to the
425 * mcast group falls to zero go ahead and
426 * tear it down. This will not free the
427 * object since we are still holding a ref
428 * from the caller
429 */
430 if (atomic_read(v: &mcg->qp_num) <= 0)
431 __rxe_destroy_mcg(mcg);
432
433 spin_unlock_bh(lock: &rxe->mcg_lock);
434 return 0;
435 }
436 }
437
438 /* we didn't find the qp on the list */
439 spin_unlock_bh(lock: &rxe->mcg_lock);
440 return -EINVAL;
441}
442
443/**
444 * rxe_attach_mcast - attach qp to multicast group (see IBA-11.3.1)
445 * @ibqp: (IB) qp object
446 * @mgid: multicast IP address
447 * @mlid: multicast LID, ignored for RoCEv2 (see IBA-A17.5.6)
448 *
449 * Returns: 0 on success else an errno
450 */
451int rxe_attach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
452{
453 int err;
454 struct rxe_dev *rxe = to_rdev(dev: ibqp->device);
455 struct rxe_qp *qp = to_rqp(qp: ibqp);
456 struct rxe_mcg *mcg;
457
458 /* takes a ref on mcg if successful */
459 mcg = rxe_get_mcg(rxe, mgid);
460 if (IS_ERR(ptr: mcg))
461 return PTR_ERR(ptr: mcg);
462
463 err = rxe_attach_mcg(mcg, qp);
464
465 /* if we failed to attach the first qp to mcg tear it down */
466 if (atomic_read(v: &mcg->qp_num) == 0)
467 rxe_destroy_mcg(mcg);
468
469 kref_put(kref: &mcg->ref_cnt, release: rxe_cleanup_mcg);
470
471 return err;
472}
473
474/**
475 * rxe_detach_mcast - detach qp from multicast group (see IBA-11.3.2)
476 * @ibqp: address of (IB) qp object
477 * @mgid: multicast IP address
478 * @mlid: multicast LID, ignored for RoCEv2 (see IBA-A17.5.6)
479 *
480 * Returns: 0 on success else an errno
481 */
482int rxe_detach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
483{
484 struct rxe_dev *rxe = to_rdev(dev: ibqp->device);
485 struct rxe_qp *qp = to_rqp(qp: ibqp);
486 struct rxe_mcg *mcg;
487 int err;
488
489 mcg = rxe_lookup_mcg(rxe, mgid);
490 if (!mcg)
491 return -EINVAL;
492
493 err = rxe_detach_mcg(mcg, qp);
494 kref_put(kref: &mcg->ref_cnt, release: rxe_cleanup_mcg);
495
496 return err;
497}
498

source code of linux/drivers/infiniband/sw/rxe/rxe_mcast.c