1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4/* Kai Shen <kaishen@linux.alibaba.com> */
5/* Copyright (c) 2020-2022, Alibaba Group. */
6
7/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8/* Fredy Neeser */
9/* Greg Joyce <greg@opengridcomputing.com> */
10/* Copyright (c) 2008-2019, IBM Corporation */
11/* Copyright (c) 2017, Open Grid Computing, Inc. */
12
13#include <linux/workqueue.h>
14#include <trace/events/sock.h>
15
16#include "erdma.h"
17#include "erdma_cm.h"
18#include "erdma_verbs.h"
19
20static struct workqueue_struct *erdma_cm_wq;
21
22static void erdma_cm_llp_state_change(struct sock *sk);
23static void erdma_cm_llp_data_ready(struct sock *sk);
24static void erdma_cm_llp_error_report(struct sock *sk);
25
26static void erdma_sk_assign_cm_upcalls(struct sock *sk)
27{
28 write_lock_bh(&sk->sk_callback_lock);
29 sk->sk_state_change = erdma_cm_llp_state_change;
30 sk->sk_data_ready = erdma_cm_llp_data_ready;
31 sk->sk_error_report = erdma_cm_llp_error_report;
32 write_unlock_bh(&sk->sk_callback_lock);
33}
34
35static void erdma_sk_save_upcalls(struct sock *sk)
36{
37 struct erdma_cep *cep = sk_to_cep(sk);
38
39 write_lock_bh(&sk->sk_callback_lock);
40 cep->sk_state_change = sk->sk_state_change;
41 cep->sk_data_ready = sk->sk_data_ready;
42 cep->sk_error_report = sk->sk_error_report;
43 write_unlock_bh(&sk->sk_callback_lock);
44}
45
46static void erdma_sk_restore_upcalls(struct sock *sk, struct erdma_cep *cep)
47{
48 sk->sk_state_change = cep->sk_state_change;
49 sk->sk_data_ready = cep->sk_data_ready;
50 sk->sk_error_report = cep->sk_error_report;
51 sk->sk_user_data = NULL;
52}
53
54static void erdma_socket_disassoc(struct socket *s)
55{
56 struct sock *sk = s->sk;
57 struct erdma_cep *cep;
58
59 if (sk) {
60 write_lock_bh(&sk->sk_callback_lock);
61 cep = sk_to_cep(sk);
62 if (cep) {
63 erdma_sk_restore_upcalls(sk, cep);
64 erdma_cep_put(ceq: cep);
65 } else {
66 WARN_ON_ONCE(1);
67 }
68 write_unlock_bh(&sk->sk_callback_lock);
69 } else {
70 WARN_ON_ONCE(1);
71 }
72}
73
74static void erdma_cep_socket_assoc(struct erdma_cep *cep, struct socket *s)
75{
76 cep->sock = s;
77 erdma_cep_get(ceq: cep);
78 s->sk->sk_user_data = cep;
79
80 erdma_sk_save_upcalls(sk: s->sk);
81 erdma_sk_assign_cm_upcalls(sk: s->sk);
82}
83
84static void erdma_disassoc_listen_cep(struct erdma_cep *cep)
85{
86 if (cep->listen_cep) {
87 erdma_cep_put(ceq: cep->listen_cep);
88 cep->listen_cep = NULL;
89 }
90}
91
92static struct erdma_cep *erdma_cep_alloc(struct erdma_dev *dev)
93{
94 struct erdma_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL);
95 unsigned long flags;
96
97 if (!cep)
98 return NULL;
99
100 INIT_LIST_HEAD(list: &cep->listenq);
101 INIT_LIST_HEAD(list: &cep->devq);
102 INIT_LIST_HEAD(list: &cep->work_freelist);
103
104 kref_init(kref: &cep->ref);
105 cep->state = ERDMA_EPSTATE_IDLE;
106 init_waitqueue_head(&cep->waitq);
107 spin_lock_init(&cep->lock);
108 cep->dev = dev;
109
110 spin_lock_irqsave(&dev->lock, flags);
111 list_add_tail(new: &cep->devq, head: &dev->cep_list);
112 spin_unlock_irqrestore(lock: &dev->lock, flags);
113
114 return cep;
115}
116
117static void erdma_cm_free_work(struct erdma_cep *cep)
118{
119 struct list_head *w, *tmp;
120 struct erdma_cm_work *work;
121
122 list_for_each_safe(w, tmp, &cep->work_freelist) {
123 work = list_entry(w, struct erdma_cm_work, list);
124 list_del(entry: &work->list);
125 kfree(objp: work);
126 }
127}
128
129static void erdma_cancel_mpatimer(struct erdma_cep *cep)
130{
131 spin_lock_bh(lock: &cep->lock);
132 if (cep->mpa_timer) {
133 if (cancel_delayed_work(dwork: &cep->mpa_timer->work)) {
134 erdma_cep_put(ceq: cep);
135 kfree(objp: cep->mpa_timer);
136 }
137 cep->mpa_timer = NULL;
138 }
139 spin_unlock_bh(lock: &cep->lock);
140}
141
142static void erdma_put_work(struct erdma_cm_work *work)
143{
144 INIT_LIST_HEAD(list: &work->list);
145 spin_lock_bh(lock: &work->cep->lock);
146 list_add(new: &work->list, head: &work->cep->work_freelist);
147 spin_unlock_bh(lock: &work->cep->lock);
148}
149
150static void erdma_cep_set_inuse(struct erdma_cep *cep)
151{
152 unsigned long flags;
153
154 spin_lock_irqsave(&cep->lock, flags);
155 while (cep->in_use) {
156 spin_unlock_irqrestore(lock: &cep->lock, flags);
157 wait_event_interruptible(cep->waitq, !cep->in_use);
158 if (signal_pending(current))
159 flush_signals(current);
160
161 spin_lock_irqsave(&cep->lock, flags);
162 }
163
164 cep->in_use = 1;
165 spin_unlock_irqrestore(lock: &cep->lock, flags);
166}
167
168static void erdma_cep_set_free(struct erdma_cep *cep)
169{
170 unsigned long flags;
171
172 spin_lock_irqsave(&cep->lock, flags);
173 cep->in_use = 0;
174 spin_unlock_irqrestore(lock: &cep->lock, flags);
175
176 wake_up(&cep->waitq);
177}
178
179static void __erdma_cep_dealloc(struct kref *ref)
180{
181 struct erdma_cep *cep = container_of(ref, struct erdma_cep, ref);
182 struct erdma_dev *dev = cep->dev;
183 unsigned long flags;
184
185 WARN_ON(cep->listen_cep);
186
187 kfree(objp: cep->private_data);
188 kfree(objp: cep->mpa.pdata);
189 spin_lock_bh(lock: &cep->lock);
190 if (!list_empty(head: &cep->work_freelist))
191 erdma_cm_free_work(cep);
192 spin_unlock_bh(lock: &cep->lock);
193
194 spin_lock_irqsave(&dev->lock, flags);
195 list_del(entry: &cep->devq);
196 spin_unlock_irqrestore(lock: &dev->lock, flags);
197 kfree(objp: cep);
198}
199
200static struct erdma_cm_work *erdma_get_work(struct erdma_cep *cep)
201{
202 struct erdma_cm_work *work = NULL;
203
204 spin_lock_bh(lock: &cep->lock);
205 if (!list_empty(head: &cep->work_freelist)) {
206 work = list_entry(cep->work_freelist.next, struct erdma_cm_work,
207 list);
208 list_del_init(entry: &work->list);
209 }
210
211 spin_unlock_bh(lock: &cep->lock);
212 return work;
213}
214
215static int erdma_cm_alloc_work(struct erdma_cep *cep, int num)
216{
217 struct erdma_cm_work *work;
218
219 while (num--) {
220 work = kmalloc(sizeof(*work), GFP_KERNEL);
221 if (!work) {
222 if (!(list_empty(head: &cep->work_freelist)))
223 erdma_cm_free_work(cep);
224 return -ENOMEM;
225 }
226 work->cep = cep;
227 INIT_LIST_HEAD(list: &work->list);
228 list_add(new: &work->list, head: &cep->work_freelist);
229 }
230
231 return 0;
232}
233
234static int erdma_cm_upcall(struct erdma_cep *cep, enum iw_cm_event_type reason,
235 int status)
236{
237 struct iw_cm_event event;
238 struct iw_cm_id *cm_id;
239
240 memset(&event, 0, sizeof(event));
241 event.status = status;
242 event.event = reason;
243
244 if (reason == IW_CM_EVENT_CONNECT_REQUEST) {
245 event.provider_data = cep;
246 cm_id = cep->listen_cep->cm_id;
247
248 event.ird = cep->dev->attrs.max_ird;
249 event.ord = cep->dev->attrs.max_ord;
250 } else {
251 cm_id = cep->cm_id;
252 }
253
254 if (reason == IW_CM_EVENT_CONNECT_REQUEST ||
255 reason == IW_CM_EVENT_CONNECT_REPLY) {
256 u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len);
257
258 if (pd_len && cep->mpa.pdata) {
259 event.private_data_len = pd_len;
260 event.private_data = cep->mpa.pdata;
261 }
262
263 getname_local(s: cep->sock, a: &event.local_addr);
264 getname_peer(s: cep->sock, a: &event.remote_addr);
265 }
266
267 return cm_id->event_handler(cm_id, &event);
268}
269
270void erdma_qp_cm_drop(struct erdma_qp *qp)
271{
272 struct erdma_cep *cep = qp->cep;
273
274 if (!qp->cep)
275 return;
276
277 erdma_cep_set_inuse(cep);
278
279 /* already closed. */
280 if (cep->state == ERDMA_EPSTATE_CLOSED)
281 goto out;
282
283 if (cep->cm_id) {
284 switch (cep->state) {
285 case ERDMA_EPSTATE_AWAIT_MPAREP:
286 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY,
287 status: -EINVAL);
288 break;
289 case ERDMA_EPSTATE_RDMA_MODE:
290 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CLOSE, status: 0);
291 break;
292 case ERDMA_EPSTATE_IDLE:
293 case ERDMA_EPSTATE_LISTENING:
294 case ERDMA_EPSTATE_CONNECTING:
295 case ERDMA_EPSTATE_AWAIT_MPAREQ:
296 case ERDMA_EPSTATE_RECVD_MPAREQ:
297 case ERDMA_EPSTATE_CLOSED:
298 default:
299 break;
300 }
301 cep->cm_id->rem_ref(cep->cm_id);
302 cep->cm_id = NULL;
303 erdma_cep_put(ceq: cep);
304 }
305 cep->state = ERDMA_EPSTATE_CLOSED;
306
307 if (cep->sock) {
308 erdma_socket_disassoc(s: cep->sock);
309 sock_release(sock: cep->sock);
310 cep->sock = NULL;
311 }
312
313 if (cep->qp) {
314 cep->qp = NULL;
315 erdma_qp_put(qp);
316 }
317out:
318 erdma_cep_set_free(cep);
319}
320
321void erdma_cep_put(struct erdma_cep *cep)
322{
323 WARN_ON(kref_read(&cep->ref) < 1);
324 kref_put(kref: &cep->ref, release: __erdma_cep_dealloc);
325}
326
327void erdma_cep_get(struct erdma_cep *cep)
328{
329 kref_get(kref: &cep->ref);
330}
331
332static int erdma_send_mpareqrep(struct erdma_cep *cep, const void *pdata,
333 u8 pd_len)
334{
335 struct socket *s = cep->sock;
336 struct mpa_rr *rr = &cep->mpa.hdr;
337 struct kvec iov[3];
338 struct msghdr msg;
339 int iovec_num = 0;
340 int ret;
341 int mpa_len;
342
343 memset(&msg, 0, sizeof(msg));
344
345 rr->params.pd_len = cpu_to_be16(pd_len);
346
347 iov[iovec_num].iov_base = rr;
348 iov[iovec_num].iov_len = sizeof(*rr);
349 iovec_num++;
350 mpa_len = sizeof(*rr);
351
352 iov[iovec_num].iov_base = &cep->mpa.ext_data;
353 iov[iovec_num].iov_len = sizeof(cep->mpa.ext_data);
354 iovec_num++;
355 mpa_len += sizeof(cep->mpa.ext_data);
356
357 if (pd_len) {
358 iov[iovec_num].iov_base = (char *)pdata;
359 iov[iovec_num].iov_len = pd_len;
360 mpa_len += pd_len;
361 iovec_num++;
362 }
363
364 ret = kernel_sendmsg(sock: s, msg: &msg, vec: iov, num: iovec_num, len: mpa_len);
365
366 return ret < 0 ? ret : 0;
367}
368
369static inline int ksock_recv(struct socket *sock, char *buf, size_t size,
370 int flags)
371{
372 struct kvec iov = { buf, size };
373 struct msghdr msg = { .msg_name = NULL, .msg_flags = flags };
374
375 return kernel_recvmsg(sock, msg: &msg, vec: &iov, num: 1, len: size, flags);
376}
377
378static int __recv_mpa_hdr(struct erdma_cep *cep, int hdr_rcvd, char *hdr,
379 int hdr_size, int *rcvd_out)
380{
381 struct socket *s = cep->sock;
382 int rcvd;
383
384 *rcvd_out = 0;
385 if (hdr_rcvd < hdr_size) {
386 rcvd = ksock_recv(sock: s, buf: hdr + hdr_rcvd, size: hdr_size - hdr_rcvd,
387 MSG_DONTWAIT);
388 if (rcvd == -EAGAIN)
389 return -EAGAIN;
390
391 if (rcvd <= 0)
392 return -ECONNABORTED;
393
394 hdr_rcvd += rcvd;
395 *rcvd_out = rcvd;
396
397 if (hdr_rcvd < hdr_size)
398 return -EAGAIN;
399 }
400
401 return 0;
402}
403
404static void __mpa_rr_set_revision(__be16 *bits, u8 rev)
405{
406 *bits = (*bits & ~MPA_RR_MASK_REVISION) |
407 (cpu_to_be16(rev) & MPA_RR_MASK_REVISION);
408}
409
410static u8 __mpa_rr_revision(__be16 mpa_rr_bits)
411{
412 __be16 rev = mpa_rr_bits & MPA_RR_MASK_REVISION;
413
414 return (u8)be16_to_cpu(rev);
415}
416
417static void __mpa_ext_set_cc(__be32 *bits, u32 cc)
418{
419 *bits = (*bits & ~MPA_EXT_FLAG_CC) |
420 (cpu_to_be32(cc) & MPA_EXT_FLAG_CC);
421}
422
423static u8 __mpa_ext_cc(__be32 mpa_ext_bits)
424{
425 __be32 cc = mpa_ext_bits & MPA_EXT_FLAG_CC;
426
427 return (u8)be32_to_cpu(cc);
428}
429
430/*
431 * Receive MPA Request/Reply header.
432 *
433 * Returns 0 if complete MPA Request/Reply haeder including
434 * eventual private data was received. Returns -EAGAIN if
435 * header was partially received or negative error code otherwise.
436 *
437 * Context: May be called in process context only
438 */
439static int erdma_recv_mpa_rr(struct erdma_cep *cep)
440{
441 struct mpa_rr *hdr = &cep->mpa.hdr;
442 struct socket *s = cep->sock;
443 u16 pd_len;
444 int rcvd, to_rcv, ret, pd_rcvd;
445
446 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) {
447 ret = __recv_mpa_hdr(cep, hdr_rcvd: cep->mpa.bytes_rcvd,
448 hdr: (char *)&cep->mpa.hdr,
449 hdr_size: sizeof(struct mpa_rr), rcvd_out: &rcvd);
450 cep->mpa.bytes_rcvd += rcvd;
451 if (ret)
452 return ret;
453 }
454
455 if (be16_to_cpu(hdr->params.pd_len) > MPA_MAX_PRIVDATA ||
456 __mpa_rr_revision(mpa_rr_bits: hdr->params.bits) != MPA_REVISION_EXT_1)
457 return -EPROTO;
458
459 if (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) <
460 sizeof(struct erdma_mpa_ext)) {
461 ret = __recv_mpa_hdr(
462 cep, hdr_rcvd: cep->mpa.bytes_rcvd - sizeof(struct mpa_rr),
463 hdr: (char *)&cep->mpa.ext_data,
464 hdr_size: sizeof(struct erdma_mpa_ext), rcvd_out: &rcvd);
465 cep->mpa.bytes_rcvd += rcvd;
466 if (ret)
467 return ret;
468 }
469
470 pd_len = be16_to_cpu(hdr->params.pd_len);
471 pd_rcvd = cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) -
472 sizeof(struct erdma_mpa_ext);
473 to_rcv = pd_len - pd_rcvd;
474
475 if (!to_rcv) {
476 /*
477 * We have received the whole MPA Request/Reply message.
478 * Check against peer protocol violation.
479 */
480 u32 word;
481
482 ret = __recv_mpa_hdr(cep, hdr_rcvd: 0, hdr: (char *)&word, hdr_size: sizeof(word),
483 rcvd_out: &rcvd);
484 if (ret == -EAGAIN && rcvd == 0)
485 return 0;
486
487 if (ret)
488 return ret;
489
490 return -EPROTO;
491 }
492
493 /*
494 * At this point, MPA header has been fully received, and pd_len != 0.
495 * So, begin to receive private data.
496 */
497 if (!cep->mpa.pdata) {
498 cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL);
499 if (!cep->mpa.pdata)
500 return -ENOMEM;
501 }
502
503 rcvd = ksock_recv(sock: s, buf: cep->mpa.pdata + pd_rcvd, size: to_rcv + 4,
504 MSG_DONTWAIT);
505 if (rcvd < 0)
506 return rcvd;
507
508 if (rcvd > to_rcv)
509 return -EPROTO;
510
511 cep->mpa.bytes_rcvd += rcvd;
512
513 if (to_rcv == rcvd)
514 return 0;
515
516 return -EAGAIN;
517}
518
519/*
520 * erdma_proc_mpareq()
521 *
522 * Read MPA Request from socket and signal new connection to IWCM
523 * if success. Caller must hold lock on corresponding listening CEP.
524 */
525static int erdma_proc_mpareq(struct erdma_cep *cep)
526{
527 struct mpa_rr *req;
528 int ret;
529
530 ret = erdma_recv_mpa_rr(cep);
531 if (ret)
532 return ret;
533
534 req = &cep->mpa.hdr;
535
536 if (memcmp(p: req->key, MPA_KEY_REQ, MPA_KEY_SIZE))
537 return -EPROTO;
538
539 memcpy(req->key, MPA_KEY_REP, MPA_KEY_SIZE);
540
541 /* Currently does not support marker and crc. */
542 if (req->params.bits & MPA_RR_FLAG_MARKERS ||
543 req->params.bits & MPA_RR_FLAG_CRC)
544 goto reject_conn;
545
546 cep->state = ERDMA_EPSTATE_RECVD_MPAREQ;
547
548 /* Keep reference until IWCM accepts/rejects */
549 erdma_cep_get(cep);
550 ret = erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REQUEST, status: 0);
551 if (ret)
552 erdma_cep_put(cep);
553
554 return ret;
555
556reject_conn:
557 req->params.bits &= ~MPA_RR_FLAG_MARKERS;
558 req->params.bits |= MPA_RR_FLAG_REJECT;
559 req->params.bits &= ~MPA_RR_FLAG_CRC;
560
561 kfree(objp: cep->mpa.pdata);
562 cep->mpa.pdata = NULL;
563 erdma_send_mpareqrep(cep, NULL, pd_len: 0);
564
565 return -EOPNOTSUPP;
566}
567
568static int erdma_proc_mpareply(struct erdma_cep *cep)
569{
570 enum erdma_qpa_mask_iwarp to_modify_attrs = 0;
571 struct erdma_mod_qp_params_iwarp params;
572 struct erdma_qp *qp = cep->qp;
573 struct mpa_rr *rep;
574 int ret;
575
576 ret = erdma_recv_mpa_rr(cep);
577 if (ret)
578 goto out_err;
579
580 erdma_cancel_mpatimer(cep);
581
582 rep = &cep->mpa.hdr;
583
584 if (memcmp(p: rep->key, MPA_KEY_REP, MPA_KEY_SIZE)) {
585 ret = -EPROTO;
586 goto out_err;
587 }
588
589 if (rep->params.bits & MPA_RR_FLAG_REJECT) {
590 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY, status: -ECONNRESET);
591 return -ECONNRESET;
592 }
593
594 /* Currently does not support marker and crc. */
595 if ((rep->params.bits & MPA_RR_FLAG_MARKERS) ||
596 (rep->params.bits & MPA_RR_FLAG_CRC)) {
597 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY, status: -ECONNREFUSED);
598 return -EINVAL;
599 }
600
601 memset(&params, 0, sizeof(params));
602 params.state = ERDMA_QPS_IWARP_RTS;
603 params.irq_size = cep->ird;
604 params.orq_size = cep->ord;
605
606 down_write(sem: &qp->state_lock);
607 if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) {
608 ret = -EINVAL;
609 up_write(sem: &qp->state_lock);
610 goto out_err;
611 }
612
613 to_modify_attrs = ERDMA_QPA_IWARP_STATE | ERDMA_QPA_IWARP_LLP_HANDLE |
614 ERDMA_QPA_IWARP_MPA | ERDMA_QPA_IWARP_IRD |
615 ERDMA_QPA_IWARP_ORD;
616
617 params.qp_type = ERDMA_QP_ACTIVE;
618 if (__mpa_ext_cc(mpa_ext_bits: cep->mpa.ext_data.bits) != qp->attrs.cc) {
619 to_modify_attrs |= ERDMA_QPA_IWARP_CC;
620 params.cc = COMPROMISE_CC;
621 }
622
623 ret = erdma_modify_qp_state_iwarp(qp, params: &params, mask: to_modify_attrs);
624
625 up_write(sem: &qp->state_lock);
626
627 if (!ret) {
628 ret = erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY, status: 0);
629 if (!ret)
630 cep->state = ERDMA_EPSTATE_RDMA_MODE;
631
632 return 0;
633 }
634
635out_err:
636 if (ret != -EAGAIN)
637 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY, status: -EINVAL);
638
639 return ret;
640}
641
642static void erdma_accept_newconn(struct erdma_cep *cep)
643{
644 struct socket *s = cep->sock;
645 struct socket *new_s = NULL;
646 struct erdma_cep *new_cep = NULL;
647 int ret = 0;
648
649 if (cep->state != ERDMA_EPSTATE_LISTENING)
650 goto error;
651
652 new_cep = erdma_cep_alloc(dev: cep->dev);
653 if (!new_cep)
654 goto error;
655
656 /*
657 * 4: Allocate a sufficient number of work elements
658 * to allow concurrent handling of local + peer close
659 * events, MPA header processing + MPA timeout.
660 */
661 if (erdma_cm_alloc_work(cep: new_cep, num: 4) != 0)
662 goto error;
663
664 /*
665 * Copy saved socket callbacks from listening CEP
666 * and assign new socket with new CEP
667 */
668 new_cep->sk_state_change = cep->sk_state_change;
669 new_cep->sk_data_ready = cep->sk_data_ready;
670 new_cep->sk_error_report = cep->sk_error_report;
671
672 ret = kernel_accept(sock: s, newsock: &new_s, O_NONBLOCK);
673 if (ret != 0)
674 goto error;
675
676 new_cep->sock = new_s;
677 erdma_cep_get(cep: new_cep);
678 new_s->sk->sk_user_data = new_cep;
679
680 tcp_sock_set_nodelay(sk: new_s->sk);
681 new_cep->state = ERDMA_EPSTATE_AWAIT_MPAREQ;
682
683 ret = erdma_cm_queue_work(ceq: new_cep, type: ERDMA_CM_WORK_MPATIMEOUT);
684 if (ret)
685 goto error;
686
687 new_cep->listen_cep = cep;
688 erdma_cep_get(cep);
689
690 if (atomic_read(v: &new_s->sk->sk_rmem_alloc)) {
691 /* MPA REQ already queued */
692 erdma_cep_set_inuse(cep: new_cep);
693 ret = erdma_proc_mpareq(cep: new_cep);
694 if (ret != -EAGAIN) {
695 erdma_cep_put(cep);
696 new_cep->listen_cep = NULL;
697 if (ret) {
698 erdma_cep_set_free(cep: new_cep);
699 goto error;
700 }
701 }
702 erdma_cep_set_free(cep: new_cep);
703 }
704 return;
705
706error:
707 if (new_cep) {
708 new_cep->state = ERDMA_EPSTATE_CLOSED;
709 erdma_cancel_mpatimer(cep: new_cep);
710
711 erdma_cep_put(cep: new_cep);
712 }
713
714 if (new_s) {
715 erdma_socket_disassoc(s: new_s);
716 sock_release(sock: new_s);
717 }
718}
719
720static int erdma_newconn_connected(struct erdma_cep *cep)
721{
722 int ret = 0;
723
724 cep->mpa.hdr.params.bits = 0;
725 __mpa_rr_set_revision(bits: &cep->mpa.hdr.params.bits, MPA_REVISION_EXT_1);
726
727 memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, MPA_KEY_SIZE);
728 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.iwarp.cookie);
729 __mpa_ext_set_cc(bits: &cep->mpa.ext_data.bits, cc: cep->qp->attrs.cc);
730
731 ret = erdma_send_mpareqrep(cep, pdata: cep->private_data, pd_len: cep->pd_len);
732 cep->state = ERDMA_EPSTATE_AWAIT_MPAREP;
733 cep->mpa.hdr.params.pd_len = 0;
734
735 if (ret >= 0)
736 ret = erdma_cm_queue_work(ceq: cep, type: ERDMA_CM_WORK_MPATIMEOUT);
737
738 return ret;
739}
740
741static void erdma_cm_work_handler(struct work_struct *w)
742{
743 struct erdma_cm_work *work;
744 struct erdma_cep *cep;
745 int release_cep = 0, ret = 0;
746
747 work = container_of(w, struct erdma_cm_work, work.work);
748 cep = work->cep;
749
750 erdma_cep_set_inuse(cep);
751
752 switch (work->type) {
753 case ERDMA_CM_WORK_CONNECTED:
754 erdma_cancel_mpatimer(cep);
755 if (cep->state == ERDMA_EPSTATE_CONNECTING) {
756 ret = erdma_newconn_connected(cep);
757 if (ret) {
758 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY,
759 status: -EIO);
760 release_cep = 1;
761 }
762 }
763 break;
764 case ERDMA_CM_WORK_CONNECTTIMEOUT:
765 if (cep->state == ERDMA_EPSTATE_CONNECTING) {
766 cep->mpa_timer = NULL;
767 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY,
768 status: -ETIMEDOUT);
769 release_cep = 1;
770 }
771 break;
772 case ERDMA_CM_WORK_ACCEPT:
773 erdma_accept_newconn(cep);
774 break;
775 case ERDMA_CM_WORK_READ_MPAHDR:
776 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
777 if (cep->listen_cep) {
778 erdma_cep_set_inuse(cep: cep->listen_cep);
779
780 if (cep->listen_cep->state ==
781 ERDMA_EPSTATE_LISTENING)
782 ret = erdma_proc_mpareq(cep);
783 else
784 ret = -EFAULT;
785
786 erdma_cep_set_free(cep: cep->listen_cep);
787
788 if (ret != -EAGAIN) {
789 erdma_cep_put(cep: cep->listen_cep);
790 cep->listen_cep = NULL;
791 if (ret)
792 erdma_cep_put(cep);
793 }
794 }
795 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
796 ret = erdma_proc_mpareply(cep);
797 }
798
799 if (ret && ret != -EAGAIN)
800 release_cep = 1;
801 break;
802 case ERDMA_CM_WORK_CLOSE_LLP:
803 if (cep->cm_id)
804 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CLOSE, status: 0);
805 release_cep = 1;
806 break;
807 case ERDMA_CM_WORK_PEER_CLOSE:
808 if (cep->cm_id) {
809 if (cep->state == ERDMA_EPSTATE_CONNECTING ||
810 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
811 /*
812 * MPA reply not received, but connection drop
813 */
814 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY,
815 status: -ECONNRESET);
816 } else if (cep->state == ERDMA_EPSTATE_RDMA_MODE) {
817 /*
818 * NOTE: IW_CM_EVENT_DISCONNECT is given just
819 * to transition IWCM into CLOSING.
820 */
821 erdma_cm_upcall(cep, reason: IW_CM_EVENT_DISCONNECT, status: 0);
822 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CLOSE, status: 0);
823 }
824 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
825 /* Socket close before MPA request received. */
826 erdma_disassoc_listen_cep(cep);
827 erdma_cep_put(cep);
828 }
829 release_cep = 1;
830 break;
831 case ERDMA_CM_WORK_MPATIMEOUT:
832 cep->mpa_timer = NULL;
833 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) {
834 /*
835 * MPA request timed out:
836 * Hide any partially received private data and signal
837 * timeout
838 */
839 cep->mpa.hdr.params.pd_len = 0;
840
841 if (cep->cm_id)
842 erdma_cm_upcall(cep, reason: IW_CM_EVENT_CONNECT_REPLY,
843 status: -ETIMEDOUT);
844 release_cep = 1;
845 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) {
846 /* No MPA req received after peer TCP stream setup. */
847 erdma_disassoc_listen_cep(cep);
848
849 erdma_cep_put(cep);
850 release_cep = 1;
851 }
852 break;
853 default:
854 WARN(1, "Undefined CM work type: %d\n", work->type);
855 }
856
857 if (release_cep) {
858 erdma_cancel_mpatimer(cep);
859 cep->state = ERDMA_EPSTATE_CLOSED;
860 if (cep->qp) {
861 struct erdma_qp *qp = cep->qp;
862 /*
863 * Serialize a potential race with application
864 * closing the QP and calling erdma_qp_cm_drop()
865 */
866 erdma_qp_get(qp);
867 erdma_cep_set_free(cep);
868
869 erdma_qp_llp_close(qp);
870 erdma_qp_put(qp);
871
872 erdma_cep_set_inuse(cep);
873 cep->qp = NULL;
874 erdma_qp_put(qp);
875 }
876
877 if (cep->sock) {
878 erdma_socket_disassoc(s: cep->sock);
879 sock_release(sock: cep->sock);
880 cep->sock = NULL;
881 }
882
883 if (cep->cm_id) {
884 cep->cm_id->rem_ref(cep->cm_id);
885 cep->cm_id = NULL;
886 if (cep->state != ERDMA_EPSTATE_LISTENING)
887 erdma_cep_put(cep);
888 }
889 }
890 erdma_cep_set_free(cep);
891 erdma_put_work(work);
892 erdma_cep_put(cep);
893}
894
895int erdma_cm_queue_work(struct erdma_cep *cep, enum erdma_work_type type)
896{
897 struct erdma_cm_work *work = erdma_get_work(cep);
898 unsigned long delay = 0;
899
900 if (!work)
901 return -ENOMEM;
902
903 work->type = type;
904 work->cep = cep;
905
906 erdma_cep_get(cep);
907
908 INIT_DELAYED_WORK(&work->work, erdma_cm_work_handler);
909
910 if (type == ERDMA_CM_WORK_MPATIMEOUT) {
911 cep->mpa_timer = work;
912
913 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP)
914 delay = MPAREP_TIMEOUT;
915 else
916 delay = MPAREQ_TIMEOUT;
917 } else if (type == ERDMA_CM_WORK_CONNECTTIMEOUT) {
918 cep->mpa_timer = work;
919
920 delay = CONNECT_TIMEOUT;
921 }
922
923 queue_delayed_work(wq: erdma_cm_wq, dwork: &work->work, delay);
924
925 return 0;
926}
927
928static void erdma_cm_llp_data_ready(struct sock *sk)
929{
930 struct erdma_cep *cep;
931
932 trace_sk_data_ready(sk);
933
934 read_lock(&sk->sk_callback_lock);
935
936 cep = sk_to_cep(sk);
937 if (!cep)
938 goto out;
939
940 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ ||
941 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP)
942 erdma_cm_queue_work(cep, type: ERDMA_CM_WORK_READ_MPAHDR);
943
944out:
945 read_unlock(&sk->sk_callback_lock);
946}
947
948static void erdma_cm_llp_error_report(struct sock *sk)
949{
950 struct erdma_cep *cep = sk_to_cep(sk);
951
952 if (cep)
953 cep->sk_error_report(sk);
954}
955
956static void erdma_cm_llp_state_change(struct sock *sk)
957{
958 struct erdma_cep *cep;
959 void (*orig_state_change)(struct sock *sk);
960
961 read_lock(&sk->sk_callback_lock);
962
963 cep = sk_to_cep(sk);
964 if (!cep) {
965 read_unlock(&sk->sk_callback_lock);
966 return;
967 }
968 orig_state_change = cep->sk_state_change;
969
970 switch (sk->sk_state) {
971 case TCP_ESTABLISHED:
972 if (cep->state == ERDMA_EPSTATE_CONNECTING)
973 erdma_cm_queue_work(cep, type: ERDMA_CM_WORK_CONNECTED);
974 else
975 erdma_cm_queue_work(cep, type: ERDMA_CM_WORK_ACCEPT);
976 break;
977 case TCP_CLOSE:
978 case TCP_CLOSE_WAIT:
979 if (cep->state != ERDMA_EPSTATE_LISTENING)
980 erdma_cm_queue_work(cep, type: ERDMA_CM_WORK_PEER_CLOSE);
981 break;
982 default:
983 break;
984 }
985 read_unlock(&sk->sk_callback_lock);
986 orig_state_change(sk);
987}
988
989static int kernel_bindconnect(struct socket *s, struct sockaddr *laddr,
990 int laddrlen, struct sockaddr *raddr,
991 int raddrlen, int flags)
992{
993 int ret;
994
995 sock_set_reuseaddr(sk: s->sk);
996 ret = s->ops->bind(s, (struct sockaddr_unsized *)laddr, laddrlen);
997 if (ret)
998 return ret;
999 ret = s->ops->connect(s, (struct sockaddr_unsized *)raddr, raddrlen, flags);
1000 return ret < 0 ? ret : 0;
1001}
1002
1003int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params)
1004{
1005 struct erdma_dev *dev = to_edev(ibdev: id->device);
1006 struct erdma_qp *qp;
1007 struct erdma_cep *cep = NULL;
1008 struct socket *s = NULL;
1009 struct sockaddr *laddr = (struct sockaddr *)&id->m_local_addr;
1010 struct sockaddr *raddr = (struct sockaddr *)&id->m_remote_addr;
1011 u16 pd_len = params->private_data_len;
1012 int ret;
1013
1014 if (pd_len > MPA_MAX_PRIVDATA)
1015 return -EINVAL;
1016
1017 if (params->ird > dev->attrs.max_ird ||
1018 params->ord > dev->attrs.max_ord)
1019 return -EINVAL;
1020
1021 if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET)
1022 return -EAFNOSUPPORT;
1023
1024 qp = find_qp_by_qpn(dev, id: params->qpn);
1025 if (!qp)
1026 return -ENOENT;
1027 erdma_qp_get(qp);
1028
1029 ret = sock_create(AF_INET, type: SOCK_STREAM, IPPROTO_TCP, res: &s);
1030 if (ret < 0)
1031 goto error_put_qp;
1032
1033 cep = erdma_cep_alloc(dev);
1034 if (!cep) {
1035 ret = -ENOMEM;
1036 goto error_release_sock;
1037 }
1038
1039 erdma_cep_set_inuse(cep);
1040
1041 /* Associate QP with CEP */
1042 erdma_cep_get(cep);
1043 qp->cep = cep;
1044 cep->qp = qp;
1045
1046 /* Associate cm_id with CEP */
1047 id->add_ref(id);
1048 cep->cm_id = id;
1049
1050 /*
1051 * 6: Allocate a sufficient number of work elements
1052 * to allow concurrent handling of local + peer close
1053 * events, MPA header processing + MPA timeout, connected event
1054 * and connect timeout.
1055 */
1056 ret = erdma_cm_alloc_work(cep, num: 6);
1057 if (ret != 0) {
1058 ret = -ENOMEM;
1059 goto error_release_cep;
1060 }
1061
1062 cep->ird = params->ird;
1063 cep->ord = params->ord;
1064 cep->state = ERDMA_EPSTATE_CONNECTING;
1065
1066 erdma_cep_socket_assoc(cep, s);
1067
1068 if (pd_len) {
1069 cep->pd_len = pd_len;
1070 cep->private_data = kmalloc(pd_len, GFP_KERNEL);
1071 if (!cep->private_data) {
1072 ret = -ENOMEM;
1073 goto error_disassoc;
1074 }
1075
1076 memcpy(cep->private_data, params->private_data,
1077 params->private_data_len);
1078 }
1079
1080 ret = kernel_bindconnect(s, laddr, laddrlen: sizeof(*laddr), raddr,
1081 raddrlen: sizeof(*raddr), O_NONBLOCK);
1082 if (ret != -EINPROGRESS && ret != 0) {
1083 goto error_disassoc;
1084 } else if (ret == 0) {
1085 ret = erdma_cm_queue_work(cep, type: ERDMA_CM_WORK_CONNECTED);
1086 if (ret)
1087 goto error_disassoc;
1088 } else {
1089 ret = erdma_cm_queue_work(cep, type: ERDMA_CM_WORK_CONNECTTIMEOUT);
1090 if (ret)
1091 goto error_disassoc;
1092 }
1093
1094 erdma_cep_set_free(cep);
1095 return 0;
1096
1097error_disassoc:
1098 kfree(objp: cep->private_data);
1099 cep->private_data = NULL;
1100 cep->pd_len = 0;
1101
1102 erdma_socket_disassoc(s);
1103
1104error_release_cep:
1105 /* disassoc with cm_id */
1106 cep->cm_id = NULL;
1107 id->rem_ref(id);
1108
1109 /* disassoc with qp */
1110 qp->cep = NULL;
1111 erdma_cep_put(cep);
1112 cep->qp = NULL;
1113
1114 cep->state = ERDMA_EPSTATE_CLOSED;
1115
1116 erdma_cep_set_free(cep);
1117
1118 /* release the cep. */
1119 erdma_cep_put(cep);
1120
1121error_release_sock:
1122 if (s)
1123 sock_release(sock: s);
1124error_put_qp:
1125 erdma_qp_put(qp);
1126
1127 return ret;
1128}
1129
1130int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params)
1131{
1132 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data;
1133 struct erdma_mod_qp_params_iwarp mod_qp_params;
1134 enum erdma_qpa_mask_iwarp to_modify_attrs = 0;
1135 struct erdma_dev *dev = to_edev(ibdev: id->device);
1136 struct erdma_qp *qp;
1137 int ret;
1138
1139 erdma_cep_set_inuse(cep);
1140 erdma_cep_put(cep);
1141
1142 /* Free lingering inbound private data */
1143 if (cep->mpa.hdr.params.pd_len) {
1144 cep->mpa.hdr.params.pd_len = 0;
1145 kfree(objp: cep->mpa.pdata);
1146 cep->mpa.pdata = NULL;
1147 }
1148 erdma_cancel_mpatimer(cep);
1149
1150 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) {
1151 erdma_cep_set_free(cep);
1152 erdma_cep_put(cep);
1153
1154 return -ECONNRESET;
1155 }
1156
1157 qp = find_qp_by_qpn(dev, id: params->qpn);
1158 if (!qp)
1159 return -ENOENT;
1160 erdma_qp_get(qp);
1161
1162 down_write(sem: &qp->state_lock);
1163 if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) {
1164 ret = -EINVAL;
1165 up_write(sem: &qp->state_lock);
1166 goto error;
1167 }
1168
1169 if (params->ord > dev->attrs.max_ord ||
1170 params->ird > dev->attrs.max_ord) {
1171 ret = -EINVAL;
1172 up_write(sem: &qp->state_lock);
1173 goto error;
1174 }
1175
1176 if (params->private_data_len > MPA_MAX_PRIVDATA) {
1177 ret = -EINVAL;
1178 up_write(sem: &qp->state_lock);
1179 goto error;
1180 }
1181
1182 cep->ird = params->ird;
1183 cep->ord = params->ord;
1184
1185 cep->cm_id = id;
1186 id->add_ref(id);
1187
1188 memset(&mod_qp_params, 0, sizeof(mod_qp_params));
1189
1190 mod_qp_params.irq_size = params->ird;
1191 mod_qp_params.orq_size = params->ord;
1192 mod_qp_params.state = ERDMA_QPS_IWARP_RTS;
1193
1194 /* Associate QP with CEP */
1195 erdma_cep_get(cep);
1196 qp->cep = cep;
1197 cep->qp = qp;
1198
1199 cep->state = ERDMA_EPSTATE_RDMA_MODE;
1200
1201 mod_qp_params.qp_type = ERDMA_QP_PASSIVE;
1202 mod_qp_params.pd_len = params->private_data_len;
1203
1204 to_modify_attrs = ERDMA_QPA_IWARP_STATE | ERDMA_QPA_IWARP_ORD |
1205 ERDMA_QPA_IWARP_LLP_HANDLE | ERDMA_QPA_IWARP_IRD |
1206 ERDMA_QPA_IWARP_MPA;
1207
1208 if (qp->attrs.cc != __mpa_ext_cc(mpa_ext_bits: cep->mpa.ext_data.bits)) {
1209 to_modify_attrs |= ERDMA_QPA_IWARP_CC;
1210 mod_qp_params.cc = COMPROMISE_CC;
1211 }
1212
1213 /* move to rts */
1214 ret = erdma_modify_qp_state_iwarp(qp, params: &mod_qp_params, mask: to_modify_attrs);
1215
1216 up_write(sem: &qp->state_lock);
1217
1218 if (ret)
1219 goto error;
1220
1221 cep->mpa.ext_data.bits = 0;
1222 __mpa_ext_set_cc(bits: &cep->mpa.ext_data.bits, cc: qp->attrs.cc);
1223 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.iwarp.cookie);
1224
1225 ret = erdma_send_mpareqrep(cep, pdata: params->private_data,
1226 pd_len: params->private_data_len);
1227 if (!ret) {
1228 ret = erdma_cm_upcall(cep, reason: IW_CM_EVENT_ESTABLISHED, status: 0);
1229 if (ret)
1230 goto error;
1231
1232 erdma_cep_set_free(cep);
1233
1234 return 0;
1235 }
1236
1237error:
1238 erdma_socket_disassoc(s: cep->sock);
1239 sock_release(sock: cep->sock);
1240 cep->sock = NULL;
1241
1242 cep->state = ERDMA_EPSTATE_CLOSED;
1243
1244 if (cep->cm_id) {
1245 cep->cm_id->rem_ref(id);
1246 cep->cm_id = NULL;
1247 }
1248
1249 if (qp->cep) {
1250 erdma_cep_put(cep);
1251 qp->cep = NULL;
1252 }
1253
1254 cep->qp = NULL;
1255 erdma_qp_put(qp);
1256
1257 erdma_cep_set_free(cep);
1258 erdma_cep_put(cep);
1259
1260 return ret;
1261}
1262
1263int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen)
1264{
1265 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data;
1266
1267 erdma_cep_set_inuse(cep);
1268 erdma_cep_put(cep);
1269
1270 erdma_cancel_mpatimer(cep);
1271
1272 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) {
1273 erdma_cep_set_free(cep);
1274 erdma_cep_put(cep);
1275
1276 return -ECONNRESET;
1277 }
1278
1279 if (__mpa_rr_revision(mpa_rr_bits: cep->mpa.hdr.params.bits) == MPA_REVISION_EXT_1) {
1280 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */
1281 erdma_send_mpareqrep(cep, pdata, pd_len: plen);
1282 }
1283
1284 erdma_socket_disassoc(s: cep->sock);
1285 sock_release(sock: cep->sock);
1286 cep->sock = NULL;
1287
1288 cep->state = ERDMA_EPSTATE_CLOSED;
1289
1290 erdma_cep_set_free(cep);
1291 erdma_cep_put(cep);
1292
1293 return 0;
1294}
1295
1296int erdma_create_listen(struct iw_cm_id *id, int backlog)
1297{
1298 struct socket *s;
1299 struct erdma_cep *cep = NULL;
1300 int ret = 0;
1301 struct erdma_dev *dev = to_edev(ibdev: id->device);
1302 int addr_family = id->local_addr.ss_family;
1303 struct sockaddr_in *laddr = &to_sockaddr_in(id->local_addr);
1304
1305 if (addr_family != AF_INET)
1306 return -EAFNOSUPPORT;
1307
1308 ret = sock_create(family: addr_family, type: SOCK_STREAM, IPPROTO_TCP, res: &s);
1309 if (ret < 0)
1310 return ret;
1311
1312 sock_set_reuseaddr(sk: s->sk);
1313
1314 /* For wildcard addr, limit binding to current device only */
1315 if (ipv4_is_zeronet(addr: laddr->sin_addr.s_addr))
1316 s->sk->sk_bound_dev_if = dev->netdev->ifindex;
1317
1318 ret = s->ops->bind(s, (struct sockaddr_unsized *)laddr,
1319 sizeof(struct sockaddr_in));
1320 if (ret)
1321 goto error;
1322
1323 cep = erdma_cep_alloc(dev);
1324 if (!cep) {
1325 ret = -ENOMEM;
1326 goto error;
1327 }
1328 erdma_cep_socket_assoc(cep, s);
1329
1330 ret = erdma_cm_alloc_work(cep, num: backlog);
1331 if (ret)
1332 goto error;
1333
1334 ret = s->ops->listen(s, backlog);
1335 if (ret)
1336 goto error;
1337
1338 cep->cm_id = id;
1339 id->add_ref(id);
1340
1341 if (!id->provider_data) {
1342 id->provider_data =
1343 kmalloc(sizeof(struct list_head), GFP_KERNEL);
1344 if (!id->provider_data) {
1345 ret = -ENOMEM;
1346 goto error;
1347 }
1348 INIT_LIST_HEAD(list: (struct list_head *)id->provider_data);
1349 }
1350
1351 list_add_tail(new: &cep->listenq, head: (struct list_head *)id->provider_data);
1352 cep->state = ERDMA_EPSTATE_LISTENING;
1353
1354 return 0;
1355
1356error:
1357 if (cep) {
1358 erdma_cep_set_inuse(cep);
1359
1360 if (cep->cm_id) {
1361 cep->cm_id->rem_ref(cep->cm_id);
1362 cep->cm_id = NULL;
1363 }
1364 cep->sock = NULL;
1365 erdma_socket_disassoc(s);
1366 cep->state = ERDMA_EPSTATE_CLOSED;
1367
1368 erdma_cep_set_free(cep);
1369 erdma_cep_put(cep);
1370 }
1371 sock_release(sock: s);
1372
1373 return ret;
1374}
1375
1376static void erdma_drop_listeners(struct iw_cm_id *id)
1377{
1378 struct list_head *p, *tmp;
1379 /*
1380 * In case of a wildcard rdma_listen on a multi-homed device,
1381 * a listener's IWCM id is associated with more than one listening CEP.
1382 */
1383 list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) {
1384 struct erdma_cep *cep =
1385 list_entry(p, struct erdma_cep, listenq);
1386
1387 list_del(entry: p);
1388
1389 erdma_cep_set_inuse(cep);
1390
1391 if (cep->cm_id) {
1392 cep->cm_id->rem_ref(cep->cm_id);
1393 cep->cm_id = NULL;
1394 }
1395 if (cep->sock) {
1396 erdma_socket_disassoc(s: cep->sock);
1397 sock_release(sock: cep->sock);
1398 cep->sock = NULL;
1399 }
1400 cep->state = ERDMA_EPSTATE_CLOSED;
1401 erdma_cep_set_free(cep);
1402 erdma_cep_put(cep);
1403 }
1404}
1405
1406int erdma_destroy_listen(struct iw_cm_id *id)
1407{
1408 if (!id->provider_data)
1409 return 0;
1410
1411 erdma_drop_listeners(id);
1412 kfree(objp: id->provider_data);
1413 id->provider_data = NULL;
1414
1415 return 0;
1416}
1417
1418int erdma_cm_init(void)
1419{
1420 erdma_cm_wq = create_singlethread_workqueue("erdma_cm_wq");
1421 if (!erdma_cm_wq)
1422 return -ENOMEM;
1423
1424 return 0;
1425}
1426
1427void erdma_cm_exit(void)
1428{
1429 if (erdma_cm_wq)
1430 destroy_workqueue(wq: erdma_cm_wq);
1431}
1432

source code of linux/drivers/infiniband/hw/erdma/erdma_cm.c