1	// SPDX-License-Identifier: GPL-2.0-or-later
2	/*
3	* rio_cm - RapidIO Channelized Messaging Driver
4	*
5	* Copyright 2013-2016 Integrated Device Technology, Inc.
6	* Copyright (c) 2015, Prodrive Technologies
7	* Copyright (c) 2015, RapidIO Trade Association
8	*/
9
10	#include <linux/module.h>
11	#include <linux/kernel.h>
12	#include <linux/dma-mapping.h>
13	#include <linux/delay.h>
14	#include <linux/sched.h>
15	#include <linux/rio.h>
16	#include <linux/rio_drv.h>
17	#include <linux/slab.h>
18	#include <linux/idr.h>
19	#include <linux/interrupt.h>
20	#include <linux/cdev.h>
21	#include <linux/fs.h>
22	#include <linux/poll.h>
23	#include <linux/reboot.h>
24	#include <linux/bitops.h>
25	#include <linux/printk.h>
26	#include <linux/rio_cm_cdev.h>
27
28	#define DRV_NAME "rio_cm"
29	#define DRV_VERSION "1.0.0"
30	#define DRV_AUTHOR "Alexandre Bounine <alexandre.bounine@idt.com>"
31	#define DRV_DESC "RapidIO Channelized Messaging Driver"
32	#define DEV_NAME "rio_cm"
33
34	/* Debug output filtering masks */
35	enum {
36	DBG_NONE = 0,
37	DBG_INIT = BIT(0), /* driver init */
38	DBG_EXIT = BIT(1), /* driver exit */
39	DBG_MPORT = BIT(2), /* mport add/remove */
40	DBG_RDEV = BIT(3), /* RapidIO device add/remove */
41	DBG_CHOP = BIT(4), /* channel operations */
42	DBG_WAIT = BIT(5), /* waiting for events */
43	DBG_TX = BIT(6), /* message TX */
44	DBG_TX_EVENT = BIT(7), /* message TX event */
45	DBG_RX_DATA = BIT(8), /* inbound data messages */
46	DBG_RX_CMD = BIT(9), /* inbound REQ/ACK/NACK messages */
47	DBG_ALL = ~0,
48	};
49
50	#ifdef DEBUG
51	#define riocm_debug(level, fmt, arg...) \
52	do { \
53	if (DBG_##level & dbg_level) \
54	pr_debug(DRV_NAME ": %s " fmt "\n", \
55	__func__, ##arg); \
56	} while (0)
57	#else
58	#define riocm_debug(level, fmt, arg...) \
59	no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
60	#endif
61
62	#define riocm_warn(fmt, arg...) \
63	pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
64
65	#define riocm_error(fmt, arg...) \
66	pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
67
68
69	static int cmbox = 1;
70	module_param(cmbox, int, S_IRUGO);
71	MODULE_PARM_DESC(cmbox, "RapidIO Mailbox number (default 1)");
72
73	static int chstart = 256;
74	module_param(chstart, int, S_IRUGO);
75	MODULE_PARM_DESC(chstart,
76	"Start channel number for dynamic allocation (default 256)");
77
78	#ifdef DEBUG
79	static u32 dbg_level = DBG_NONE;
80	module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
81	MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
82	#endif
83
84	MODULE_AUTHOR(DRV_AUTHOR);
85	MODULE_DESCRIPTION(DRV_DESC);
86	MODULE_LICENSE("GPL");
87	MODULE_VERSION(DRV_VERSION);
88
89	#define RIOCM_TX_RING_SIZE 128
90	#define RIOCM_RX_RING_SIZE 128
91	#define RIOCM_CONNECT_TO 3 /* connect response TO (in sec) */
92
93	#define RIOCM_MAX_CHNUM 0xffff /* Use full range of u16 field */
94	#define RIOCM_CHNUM_AUTO 0
95	#define RIOCM_MAX_EP_COUNT 0x10000 /* Max number of endpoints */
96
97	enum rio_cm_state {
98	RIO_CM_IDLE,
99	RIO_CM_CONNECT,
100	RIO_CM_CONNECTED,
101	RIO_CM_DISCONNECT,
102	RIO_CM_CHAN_BOUND,
103	RIO_CM_LISTEN,
104	RIO_CM_DESTROYING,
105	};
106
107	enum rio_cm_pkt_type {
108	RIO_CM_SYS = 0xaa,
109	RIO_CM_CHAN = 0x55,
110	};
111
112	enum rio_cm_chop {
113	CM_CONN_REQ,
114	CM_CONN_ACK,
115	CM_CONN_CLOSE,
116	CM_DATA_MSG,
117	};
118
119	struct rio_ch_base_bhdr {
120	u32 src_id;
121	u32 dst_id;
122	#define RIO_HDR_LETTER_MASK 0xffff0000
123	#define RIO_HDR_MBOX_MASK 0x0000ffff
124	u8 src_mbox;
125	u8 dst_mbox;
126	u8 type;
127	} __attribute__((__packed__));
128
129	struct rio_ch_chan_hdr {
130	struct rio_ch_base_bhdr bhdr;
131	u8 ch_op;
132	u16 dst_ch;
133	u16 src_ch;
134	u16 msg_len;
135	u16 rsrvd;
136	} __attribute__((__packed__));
137
138	struct tx_req {
139	struct list_head node;
140	struct rio_dev *rdev;
141	void *buffer;
142	size_t len;
143	};
144
145	struct cm_dev {
146	struct list_head list;
147	struct rio_mport *mport;
148	void *rx_buf[RIOCM_RX_RING_SIZE];
149	int rx_slots;
150	struct mutex rx_lock;
151
152	void *tx_buf[RIOCM_TX_RING_SIZE];
153	int tx_slot;
154	int tx_cnt;
155	int tx_ack_slot;
156	struct list_head tx_reqs;
157	spinlock_t tx_lock;
158
159	struct list_head peers;
160	u32 npeers;
161	struct workqueue_struct *rx_wq;
162	struct work_struct rx_work;
163	};
164
165	struct chan_rx_ring {
166	void *buf[RIOCM_RX_RING_SIZE];
167	int head;
168	int tail;
169	int count;
170
171	/* Tracking RX buffers reported to upper level */
172	void *inuse[RIOCM_RX_RING_SIZE];
173	int inuse_cnt;
174	};
175
176	struct rio_channel {
177	u16 id; /* local channel ID */
178	struct kref ref; /* channel refcount */
179	struct file *filp;
180	struct cm_dev *cmdev; /* associated CM device object */
181	struct rio_dev *rdev; /* remote RapidIO device */
182	enum rio_cm_state state;
183	int error;
184	spinlock_t lock;
185	void *context;
186	u32 loc_destid; /* local destID */
187	u32 rem_destid; /* remote destID */
188	u16 rem_channel; /* remote channel ID */
189	struct list_head accept_queue;
190	struct list_head ch_node;
191	struct completion comp;
192	struct completion comp_close;
193	struct chan_rx_ring rx_ring;
194	};
195
196	struct cm_peer {
197	struct list_head node;
198	struct rio_dev *rdev;
199	};
200
201	struct rio_cm_work {
202	struct work_struct work;
203	struct cm_dev *cm;
204	void *data;
205	};
206
207	struct conn_req {
208	struct list_head node;
209	u32 destid; /* requester destID */
210	u16 chan; /* requester channel ID */
211	struct cm_dev *cmdev;
212	};
213
214	/*
215	* A channel_dev structure represents a CM_CDEV
216	* @cdev Character device
217	* @dev Associated device object
218	*/
219	struct channel_dev {
220	struct cdev cdev;
221	struct device *dev;
222	};
223
224	static struct rio_channel *riocm_ch_alloc(u16 ch_num);
225	static void riocm_ch_free(struct kref *ref);
226	static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
227	void *buffer, size_t len);
228	static int riocm_ch_close(struct rio_channel *ch);
229
230	static DEFINE_SPINLOCK(idr_lock);
231	static DEFINE_IDR(ch_idr);
232
233	static LIST_HEAD(cm_dev_list);
234	static DECLARE_RWSEM(rdev_sem);
235
236	static const struct class dev_class = {
237	.name = DRV_NAME,
238	};
239	static unsigned int dev_major;
240	static unsigned int dev_minor_base;
241	static dev_t dev_number;
242	static struct channel_dev riocm_cdev;
243
244	#define is_msg_capable(src_ops, dst_ops) \
245	((src_ops & RIO_SRC_OPS_DATA_MSG) && \
246	(dst_ops & RIO_DST_OPS_DATA_MSG))
247	#define dev_cm_capable(dev) \
248	is_msg_capable(dev->src_ops, dev->dst_ops)
249
250	static int riocm_cmp(struct rio_channel *ch, enum rio_cm_state cmp)
251	{
252	int ret;
253
254	spin_lock_bh(lock: &ch->lock);
255	ret = (ch->state == cmp);
256	spin_unlock_bh(lock: &ch->lock);
257	return ret;
258	}
259
260	static int riocm_cmp_exch(struct rio_channel *ch,
261	enum rio_cm_state cmp, enum rio_cm_state exch)
262	{
263	int ret;
264
265	spin_lock_bh(lock: &ch->lock);
266	ret = (ch->state == cmp);
267	if (ret)
268	ch->state = exch;
269	spin_unlock_bh(lock: &ch->lock);
270	return ret;
271	}
272
273	static enum rio_cm_state riocm_exch(struct rio_channel *ch,
274	enum rio_cm_state exch)
275	{
276	enum rio_cm_state old;
277
278	spin_lock_bh(lock: &ch->lock);
279	old = ch->state;
280	ch->state = exch;
281	spin_unlock_bh(lock: &ch->lock);
282	return old;
283	}
284
285	static struct rio_channel *riocm_get_channel(u16 nr)
286	{
287	struct rio_channel *ch;
288
289	spin_lock_bh(lock: &idr_lock);
290	ch = idr_find(&ch_idr, id: nr);
291	if (ch)
292	kref_get(kref: &ch->ref);
293	spin_unlock_bh(lock: &idr_lock);
294	return ch;
295	}
296
297	static void riocm_put_channel(struct rio_channel *ch)
298	{
299	kref_put(kref: &ch->ref, release: riocm_ch_free);
300	}
301
302	static void *riocm_rx_get_msg(struct cm_dev *cm)
303	{
304	void *msg;
305	int i;
306
307	msg = rio_get_inb_message(mport: cm->mport, mbox: cmbox);
308	if (msg) {
309	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
310	if (cm->rx_buf[i] == msg) {
311	cm->rx_buf[i] = NULL;
312	cm->rx_slots++;
313	break;
314	}
315	}
316
317	if (i == RIOCM_RX_RING_SIZE)
318	riocm_warn("no record for buffer 0x%p", msg);
319	}
320
321	return msg;
322	}
323
324	/*
325	* riocm_rx_fill - fills a ring of receive buffers for given cm device
326	* @cm: cm_dev object
327	* @nent: max number of entries to fill
328	*
329	* Returns: none
330	*/
331	static void riocm_rx_fill(struct cm_dev *cm, int nent)
332	{
333	int i;
334
335	if (cm->rx_slots == 0)
336	return;
337
338	for (i = 0; i < RIOCM_RX_RING_SIZE && cm->rx_slots && nent; i++) {
339	if (cm->rx_buf[i] == NULL) {
340	cm->rx_buf[i] = kmalloc(RIO_MAX_MSG_SIZE, GFP_KERNEL);
341	if (cm->rx_buf[i] == NULL)
342	break;
343	rio_add_inb_buffer(mport: cm->mport, mbox: cmbox, buffer: cm->rx_buf[i]);
344	cm->rx_slots--;
345	nent--;
346	}
347	}
348	}
349
350	/*
351	* riocm_rx_free - frees all receive buffers associated with given cm device
352	* @cm: cm_dev object
353	*
354	* Returns: none
355	*/
356	static void riocm_rx_free(struct cm_dev *cm)
357	{
358	int i;
359
360	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
361	if (cm->rx_buf[i] != NULL) {
362	kfree(objp: cm->rx_buf[i]);
363	cm->rx_buf[i] = NULL;
364	}
365	}
366	}
367
368	/*
369	* riocm_req_handler - connection request handler
370	* @cm: cm_dev object
371	* @req_data: pointer to the request packet
372	*
373	* Returns: 0 if success, or
374	* -EINVAL if channel is not in correct state,
375	* -ENODEV if cannot find a channel with specified ID,
376	* -ENOMEM if unable to allocate memory to store the request
377	*/
378	static int riocm_req_handler(struct cm_dev *cm, void *req_data)
379	{
380	struct rio_channel *ch;
381	struct conn_req *req;
382	struct rio_ch_chan_hdr *hh = req_data;
383	u16 chnum;
384
385	chnum = ntohs(hh->dst_ch);
386
387	ch = riocm_get_channel(nr: chnum);
388
389	if (!ch)
390	return -ENODEV;
391
392	if (ch->state != RIO_CM_LISTEN) {
393	riocm_debug(RX_CMD, "channel %d is not in listen state", chnum);
394	riocm_put_channel(ch);
395	return -EINVAL;
396	}
397
398	req = kzalloc(size: sizeof(*req), GFP_KERNEL);
399	if (!req) {
400	riocm_put_channel(ch);
401	return -ENOMEM;
402	}
403
404	req->destid = ntohl(hh->bhdr.src_id);
405	req->chan = ntohs(hh->src_ch);
406	req->cmdev = cm;
407
408	spin_lock_bh(lock: &ch->lock);
409	list_add_tail(new: &req->node, head: &ch->accept_queue);
410	spin_unlock_bh(lock: &ch->lock);
411	complete(&ch->comp);
412	riocm_put_channel(ch);
413
414	return 0;
415	}
416
417	/*
418	* riocm_resp_handler - response to connection request handler
419	* @resp_data: pointer to the response packet
420	*
421	* Returns: 0 if success, or
422	* -EINVAL if channel is not in correct state,
423	* -ENODEV if cannot find a channel with specified ID,
424	*/
425	static int riocm_resp_handler(void *resp_data)
426	{
427	struct rio_channel *ch;
428	struct rio_ch_chan_hdr *hh = resp_data;
429	u16 chnum;
430
431	chnum = ntohs(hh->dst_ch);
432	ch = riocm_get_channel(nr: chnum);
433	if (!ch)
434	return -ENODEV;
435
436	if (ch->state != RIO_CM_CONNECT) {
437	riocm_put_channel(ch);
438	return -EINVAL;
439	}
440
441	riocm_exch(ch, exch: RIO_CM_CONNECTED);
442	ch->rem_channel = ntohs(hh->src_ch);
443	complete(&ch->comp);
444	riocm_put_channel(ch);
445
446	return 0;
447	}
448
449	/*
450	* riocm_close_handler - channel close request handler
451	* @req_data: pointer to the request packet
452	*
453	* Returns: 0 if success, or
454	* -ENODEV if cannot find a channel with specified ID,
455	* + error codes returned by riocm_ch_close.
456	*/
457	static int riocm_close_handler(void *data)
458	{
459	struct rio_channel *ch;
460	struct rio_ch_chan_hdr *hh = data;
461	int ret;
462
463	riocm_debug(RX_CMD, "for ch=%d", ntohs(hh->dst_ch));
464
465	spin_lock_bh(lock: &idr_lock);
466	ch = idr_find(&ch_idr, ntohs(hh->dst_ch));
467	if (!ch) {
468	spin_unlock_bh(lock: &idr_lock);
469	return -ENODEV;
470	}
471	idr_remove(&ch_idr, id: ch->id);
472	spin_unlock_bh(lock: &idr_lock);
473
474	riocm_exch(ch, exch: RIO_CM_DISCONNECT);
475
476	ret = riocm_ch_close(ch);
477	if (ret)
478	riocm_debug(RX_CMD, "riocm_ch_close() returned %d", ret);
479
480	return 0;
481	}
482
483	/*
484	* rio_cm_handler - function that services request (non-data) packets
485	* @cm: cm_dev object
486	* @data: pointer to the packet
487	*/
488	static void rio_cm_handler(struct cm_dev *cm, void *data)
489	{
490	struct rio_ch_chan_hdr *hdr;
491
492	if (!rio_mport_is_running(mport: cm->mport))
493	goto out;
494
495	hdr = data;
496
497	riocm_debug(RX_CMD, "OP=%x for ch=%d from %d",
498	hdr->ch_op, ntohs(hdr->dst_ch), ntohs(hdr->src_ch));
499
500	switch (hdr->ch_op) {
501	case CM_CONN_REQ:
502	riocm_req_handler(cm, req_data: data);
503	break;
504	case CM_CONN_ACK:
505	riocm_resp_handler(resp_data: data);
506	break;
507	case CM_CONN_CLOSE:
508	riocm_close_handler(data);
509	break;
510	default:
511	riocm_error("Invalid packet header");
512	break;
513	}
514	out:
515	kfree(objp: data);
516	}
517
518	/*
519	* rio_rx_data_handler - received data packet handler
520	* @cm: cm_dev object
521	* @buf: data packet
522	*
523	* Returns: 0 if success, or
524	* -ENODEV if cannot find a channel with specified ID,
525	* -EIO if channel is not in CONNECTED state,
526	* -ENOMEM if channel RX queue is full (packet discarded)
527	*/
528	static int rio_rx_data_handler(struct cm_dev *cm, void *buf)
529	{
530	struct rio_ch_chan_hdr *hdr;
531	struct rio_channel *ch;
532
533	hdr = buf;
534
535	riocm_debug(RX_DATA, "for ch=%d", ntohs(hdr->dst_ch));
536
537	ch = riocm_get_channel(ntohs(hdr->dst_ch));
538	if (!ch) {
539	/* Discard data message for non-existing channel */
540	kfree(objp: buf);
541	return -ENODEV;
542	}
543
544	/* Place pointer to the buffer into channel's RX queue */
545	spin_lock(lock: &ch->lock);
546
547	if (ch->state != RIO_CM_CONNECTED) {
548	/* Channel is not ready to receive data, discard a packet */
549	riocm_debug(RX_DATA, "ch=%d is in wrong state=%d",
550	ch->id, ch->state);
551	spin_unlock(lock: &ch->lock);
552	kfree(objp: buf);
553	riocm_put_channel(ch);
554	return -EIO;
555	}
556
557	if (ch->rx_ring.count == RIOCM_RX_RING_SIZE) {
558	/* If RX ring is full, discard a packet */
559	riocm_debug(RX_DATA, "ch=%d is full", ch->id);
560	spin_unlock(lock: &ch->lock);
561	kfree(objp: buf);
562	riocm_put_channel(ch);
563	return -ENOMEM;
564	}
565
566	ch->rx_ring.buf[ch->rx_ring.head] = buf;
567	ch->rx_ring.head++;
568	ch->rx_ring.count++;
569	ch->rx_ring.head %= RIOCM_RX_RING_SIZE;
570
571	complete(&ch->comp);
572
573	spin_unlock(lock: &ch->lock);
574	riocm_put_channel(ch);
575
576	return 0;
577	}
578
579	/*
580	* rio_ibmsg_handler - inbound message packet handler
581	*/
582	static void rio_ibmsg_handler(struct work_struct *work)
583	{
584	struct cm_dev *cm = container_of(work, struct cm_dev, rx_work);
585	void *data;
586	struct rio_ch_chan_hdr *hdr;
587
588	if (!rio_mport_is_running(mport: cm->mport))
589	return;
590
591	while (1) {
592	mutex_lock(&cm->rx_lock);
593	data = riocm_rx_get_msg(cm);
594	if (data)
595	riocm_rx_fill(cm, nent: 1);
596	mutex_unlock(lock: &cm->rx_lock);
597
598	if (data == NULL)
599	break;
600
601	hdr = data;
602
603	if (hdr->bhdr.type != RIO_CM_CHAN) {
604	/* For now simply discard packets other than channel */
605	riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
606	hdr->bhdr.type);
607	kfree(objp: data);
608	continue;
609	}
610
611	/* Process a channel message */
612	if (hdr->ch_op == CM_DATA_MSG)
613	rio_rx_data_handler(cm, buf: data);
614	else
615	rio_cm_handler(cm, data);
616	}
617	}
618
619	static void riocm_inb_msg_event(struct rio_mport *mport, void *dev_id,
620	int mbox, int slot)
621	{
622	struct cm_dev *cm = dev_id;
623
624	if (rio_mport_is_running(mport: cm->mport) && !work_pending(&cm->rx_work))
625	queue_work(wq: cm->rx_wq, work: &cm->rx_work);
626	}
627
628	/*
629	* rio_txcq_handler - TX completion handler
630	* @cm: cm_dev object
631	* @slot: TX queue slot
632	*
633	* TX completion handler also ensures that pending request packets are placed
634	* into transmit queue as soon as a free slot becomes available. This is done
635	* to give higher priority to request packets during high intensity data flow.
636	*/
637	static void rio_txcq_handler(struct cm_dev *cm, int slot)
638	{
639	int ack_slot;
640
641	/* ATTN: Add TX completion notification if/when direct buffer
642	* transfer is implemented. At this moment only correct tracking
643	* of tx_count is important.
644	*/
645	riocm_debug(TX_EVENT, "for mport_%d slot %d tx_cnt %d",
646	cm->mport->id, slot, cm->tx_cnt);
647
648	spin_lock(lock: &cm->tx_lock);
649	ack_slot = cm->tx_ack_slot;
650
651	if (ack_slot == slot)
652	riocm_debug(TX_EVENT, "slot == ack_slot");
653
654	while (cm->tx_cnt && ((ack_slot != slot) ||
655	(cm->tx_cnt == RIOCM_TX_RING_SIZE))) {
656
657	cm->tx_buf[ack_slot] = NULL;
658	++ack_slot;
659	ack_slot &= (RIOCM_TX_RING_SIZE - 1);
660	cm->tx_cnt--;
661	}
662
663	if (cm->tx_cnt < 0 || cm->tx_cnt > RIOCM_TX_RING_SIZE)
664	riocm_error("tx_cnt %d out of sync", cm->tx_cnt);
665
666	WARN_ON((cm->tx_cnt < 0) || (cm->tx_cnt > RIOCM_TX_RING_SIZE));
667
668	cm->tx_ack_slot = ack_slot;
669
670	/*
671	* If there are pending requests, insert them into transmit queue
672	*/
673	if (!list_empty(head: &cm->tx_reqs) && (cm->tx_cnt < RIOCM_TX_RING_SIZE)) {
674	struct tx_req *req, *_req;
675	int rc;
676
677	list_for_each_entry_safe(req, _req, &cm->tx_reqs, node) {
678	list_del(entry: &req->node);
679	cm->tx_buf[cm->tx_slot] = req->buffer;
680	rc = rio_add_outb_message(mport: cm->mport, rdev: req->rdev, mbox: cmbox,
681	buffer: req->buffer, len: req->len);
682	kfree(objp: req->buffer);
683	kfree(objp: req);
684
685	++cm->tx_cnt;
686	++cm->tx_slot;
687	cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
688	if (cm->tx_cnt == RIOCM_TX_RING_SIZE)
689	break;
690	}
691	}
692
693	spin_unlock(lock: &cm->tx_lock);
694	}
695
696	static void riocm_outb_msg_event(struct rio_mport *mport, void *dev_id,
697	int mbox, int slot)
698	{
699	struct cm_dev *cm = dev_id;
700
701	if (cm && rio_mport_is_running(mport: cm->mport))
702	rio_txcq_handler(cm, slot);
703	}
704
705	static int riocm_queue_req(struct cm_dev *cm, struct rio_dev *rdev,
706	void *buffer, size_t len)
707	{
708	unsigned long flags;
709	struct tx_req *treq;
710
711	treq = kzalloc(size: sizeof(*treq), GFP_KERNEL);
712	if (treq == NULL)
713	return -ENOMEM;
714
715	treq->rdev = rdev;
716	treq->buffer = buffer;
717	treq->len = len;
718
719	spin_lock_irqsave(&cm->tx_lock, flags);
720	list_add_tail(new: &treq->node, head: &cm->tx_reqs);
721	spin_unlock_irqrestore(lock: &cm->tx_lock, flags);
722	return 0;
723	}
724
725	/*
726	* riocm_post_send - helper function that places packet into msg TX queue
727	* @cm: cm_dev object
728	* @rdev: target RapidIO device object (required by outbound msg interface)
729	* @buffer: pointer to a packet buffer to send
730	* @len: length of data to transfer
731	* @req: request priority flag
732	*
733	* Returns: 0 if success, or error code otherwise.
734	*/
735	static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
736	void *buffer, size_t len)
737	{
738	int rc;
739	unsigned long flags;
740
741	spin_lock_irqsave(&cm->tx_lock, flags);
742
743	if (cm->mport == NULL) {
744	rc = -ENODEV;
745	goto err_out;
746	}
747
748	if (cm->tx_cnt == RIOCM_TX_RING_SIZE) {
749	riocm_debug(TX, "Tx Queue is full");
750	rc = -EBUSY;
751	goto err_out;
752	}
753
754	cm->tx_buf[cm->tx_slot] = buffer;
755	rc = rio_add_outb_message(mport: cm->mport, rdev, mbox: cmbox, buffer, len);
756
757	riocm_debug(TX, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
758	buffer, rdev->destid, cm->tx_slot, cm->tx_cnt);
759
760	++cm->tx_cnt;
761	++cm->tx_slot;
762	cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
763
764	err_out:
765	spin_unlock_irqrestore(lock: &cm->tx_lock, flags);
766	return rc;
767	}
768
769	/*
770	* riocm_ch_send - sends a data packet to a remote device
771	* @ch_id: local channel ID
772	* @buf: pointer to a data buffer to send (including CM header)
773	* @len: length of data to transfer (including CM header)
774	*
775	* ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
776	*
777	* Returns: 0 if success, or
778	* -EINVAL if one or more input parameters is/are not valid,
779	* -ENODEV if cannot find a channel with specified ID,
780	* -EAGAIN if a channel is not in CONNECTED state,
781	* + error codes returned by HW send routine.
782	*/
783	static int riocm_ch_send(u16 ch_id, void *buf, int len)
784	{
785	struct rio_channel *ch;
786	struct rio_ch_chan_hdr *hdr;
787	int ret;
788
789	if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE)
790	return -EINVAL;
791
792	ch = riocm_get_channel(nr: ch_id);
793	if (!ch) {
794	riocm_error("%s(%d) ch_%d not found", current->comm,
795	task_pid_nr(current), ch_id);
796	return -ENODEV;
797	}
798
799	if (!riocm_cmp(ch, cmp: RIO_CM_CONNECTED)) {
800	ret = -EAGAIN;
801	goto err_out;
802	}
803
804	/*
805	* Fill buffer header section with corresponding channel data
806	*/
807	hdr = buf;
808
809	hdr->bhdr.src_id = htonl(ch->loc_destid);
810	hdr->bhdr.dst_id = htonl(ch->rem_destid);
811	hdr->bhdr.src_mbox = cmbox;
812	hdr->bhdr.dst_mbox = cmbox;
813	hdr->bhdr.type = RIO_CM_CHAN;
814	hdr->ch_op = CM_DATA_MSG;
815	hdr->dst_ch = htons(ch->rem_channel);
816	hdr->src_ch = htons(ch->id);
817	hdr->msg_len = htons((u16)len);
818
819	/* ATTN: the function call below relies on the fact that underlying
820	* HW-specific add_outb_message() routine copies TX data into its own
821	* internal transfer buffer (true for all RIONET compatible mport
822	* drivers). Must be reviewed if mport driver uses the buffer directly.
823	*/
824
825	ret = riocm_post_send(cm: ch->cmdev, rdev: ch->rdev, buffer: buf, len);
826	if (ret)
827	riocm_debug(TX, "ch %d send_err=%d", ch->id, ret);
828	err_out:
829	riocm_put_channel(ch);
830	return ret;
831	}
832
833	static int riocm_ch_free_rxbuf(struct rio_channel *ch, void *buf)
834	{
835	int i, ret = -EINVAL;
836
837	spin_lock_bh(lock: &ch->lock);
838
839	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
840	if (ch->rx_ring.inuse[i] == buf) {
841	ch->rx_ring.inuse[i] = NULL;
842	ch->rx_ring.inuse_cnt--;
843	ret = 0;
844	break;
845	}
846	}
847
848	spin_unlock_bh(lock: &ch->lock);
849
850	if (!ret)
851	kfree(objp: buf);
852
853	return ret;
854	}
855
856	/*
857	* riocm_ch_receive - fetch a data packet received for the specified channel
858	* @ch: local channel ID
859	* @buf: pointer to a packet buffer
860	* @timeout: timeout to wait for incoming packet (in jiffies)
861	*
862	* Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
863	* -EAGAIN if a channel is not in CONNECTED state,
864	* -ENOMEM if in-use tracking queue is full,
865	* -ETIME if wait timeout expired,
866	* -EINTR if wait was interrupted.
867	*/
868	static int riocm_ch_receive(struct rio_channel *ch, void **buf, long timeout)
869	{
870	void *rxmsg = NULL;
871	int i, ret = 0;
872	long wret;
873
874	if (!riocm_cmp(ch, cmp: RIO_CM_CONNECTED)) {
875	ret = -EAGAIN;
876	goto out;
877	}
878
879	if (ch->rx_ring.inuse_cnt == RIOCM_RX_RING_SIZE) {
880	/* If we do not have entries to track buffers given to upper
881	* layer, reject request.
882	*/
883	ret = -ENOMEM;
884	goto out;
885	}
886
887	wret = wait_for_completion_interruptible_timeout(x: &ch->comp, timeout);
888
889	riocm_debug(WAIT, "wait on %d returned %ld", ch->id, wret);
890
891	if (!wret)
892	ret = -ETIME;
893	else if (wret == -ERESTARTSYS)
894	ret = -EINTR;
895	else
896	ret = riocm_cmp(ch, cmp: RIO_CM_CONNECTED) ? 0 : -ECONNRESET;
897
898	if (ret)
899	goto out;
900
901	spin_lock_bh(lock: &ch->lock);
902
903	rxmsg = ch->rx_ring.buf[ch->rx_ring.tail];
904	ch->rx_ring.buf[ch->rx_ring.tail] = NULL;
905	ch->rx_ring.count--;
906	ch->rx_ring.tail++;
907	ch->rx_ring.tail %= RIOCM_RX_RING_SIZE;
908	ret = -ENOMEM;
909
910	for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
911	if (ch->rx_ring.inuse[i] == NULL) {
912	ch->rx_ring.inuse[i] = rxmsg;
913	ch->rx_ring.inuse_cnt++;
914	ret = 0;
915	break;
916	}
917	}
918
919	if (ret) {
920	/* We have no entry to store pending message: drop it */
921	kfree(objp: rxmsg);
922	rxmsg = NULL;
923	}
924
925	spin_unlock_bh(lock: &ch->lock);
926	out:
927	*buf = rxmsg;
928	return ret;
929	}
930
931	/*
932	* riocm_ch_connect - sends a connect request to a remote device
933	* @loc_ch: local channel ID
934	* @cm: CM device to send connect request
935	* @peer: target RapidIO device
936	* @rem_ch: remote channel ID
937	*
938	* Returns: 0 if success, or
939	* -EINVAL if the channel is not in IDLE state,
940	* -EAGAIN if no connection request available immediately,
941	* -ETIME if ACK response timeout expired,
942	* -EINTR if wait for response was interrupted.
943	*/
944	static int riocm_ch_connect(u16 loc_ch, struct cm_dev *cm,
945	struct cm_peer *peer, u16 rem_ch)
946	{
947	struct rio_channel *ch = NULL;
948	struct rio_ch_chan_hdr *hdr;
949	int ret;
950	long wret;
951
952	ch = riocm_get_channel(nr: loc_ch);
953	if (!ch)
954	return -ENODEV;
955
956	if (!riocm_cmp_exch(ch, cmp: RIO_CM_IDLE, exch: RIO_CM_CONNECT)) {
957	ret = -EINVAL;
958	goto conn_done;
959	}
960
961	ch->cmdev = cm;
962	ch->rdev = peer->rdev;
963	ch->context = NULL;
964	ch->loc_destid = cm->mport->host_deviceid;
965	ch->rem_channel = rem_ch;
966
967	/*
968	* Send connect request to the remote RapidIO device
969	*/
970
971	hdr = kzalloc(size: sizeof(*hdr), GFP_KERNEL);
972	if (hdr == NULL) {
973	ret = -ENOMEM;
974	goto conn_done;
975	}
976
977	hdr->bhdr.src_id = htonl(ch->loc_destid);
978	hdr->bhdr.dst_id = htonl(peer->rdev->destid);
979	hdr->bhdr.src_mbox = cmbox;
980	hdr->bhdr.dst_mbox = cmbox;
981	hdr->bhdr.type = RIO_CM_CHAN;
982	hdr->ch_op = CM_CONN_REQ;
983	hdr->dst_ch = htons(rem_ch);
984	hdr->src_ch = htons(loc_ch);
985
986	/* ATTN: the function call below relies on the fact that underlying
987	* HW-specific add_outb_message() routine copies TX data into its
988	* internal transfer buffer. Must be reviewed if mport driver uses
989	* this buffer directly.
990	*/
991	ret = riocm_post_send(cm, rdev: peer->rdev, buffer: hdr, len: sizeof(*hdr));
992
993	if (ret != -EBUSY) {
994	kfree(objp: hdr);
995	} else {
996	ret = riocm_queue_req(cm, rdev: peer->rdev, buffer: hdr, len: sizeof(*hdr));
997	if (ret)
998	kfree(objp: hdr);
999	}
1000
1001	if (ret) {
1002	riocm_cmp_exch(ch, cmp: RIO_CM_CONNECT, exch: RIO_CM_IDLE);
1003	goto conn_done;
1004	}
1005
1006	/* Wait for connect response from the remote device */
1007	wret = wait_for_completion_interruptible_timeout(x: &ch->comp,
1008	RIOCM_CONNECT_TO * HZ);
1009	riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1010
1011	if (!wret)
1012	ret = -ETIME;
1013	else if (wret == -ERESTARTSYS)
1014	ret = -EINTR;
1015	else
1016	ret = riocm_cmp(ch, cmp: RIO_CM_CONNECTED) ? 0 : -1;
1017
1018	conn_done:
1019	riocm_put_channel(ch);
1020	return ret;
1021	}
1022
1023	static int riocm_send_ack(struct rio_channel *ch)
1024	{
1025	struct rio_ch_chan_hdr *hdr;
1026	int ret;
1027
1028	hdr = kzalloc(size: sizeof(*hdr), GFP_KERNEL);
1029	if (hdr == NULL)
1030	return -ENOMEM;
1031
1032	hdr->bhdr.src_id = htonl(ch->loc_destid);
1033	hdr->bhdr.dst_id = htonl(ch->rem_destid);
1034	hdr->dst_ch = htons(ch->rem_channel);
1035	hdr->src_ch = htons(ch->id);
1036	hdr->bhdr.src_mbox = cmbox;
1037	hdr->bhdr.dst_mbox = cmbox;
1038	hdr->bhdr.type = RIO_CM_CHAN;
1039	hdr->ch_op = CM_CONN_ACK;
1040
1041	/* ATTN: the function call below relies on the fact that underlying
1042	* add_outb_message() routine copies TX data into its internal transfer
1043	* buffer. Review if switching to direct buffer version.
1044	*/
1045	ret = riocm_post_send(cm: ch->cmdev, rdev: ch->rdev, buffer: hdr, len: sizeof(*hdr));
1046
1047	if (ret == -EBUSY && !riocm_queue_req(cm: ch->cmdev,
1048	rdev: ch->rdev, buffer: hdr, len: sizeof(*hdr)))
1049	return 0;
1050	kfree(objp: hdr);
1051
1052	if (ret)
1053	riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1054	ch->id, rio_name(ch->rdev), ret);
1055	return ret;
1056	}
1057
1058	/*
1059	* riocm_ch_accept - accept incoming connection request
1060	* @ch_id: channel ID
1061	* @new_ch_id: local mport device
1062	* @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1063	* request is not available).
1064	*
1065	* Returns: pointer to new channel struct if success, or error-valued pointer:
1066	* -ENODEV - cannot find specified channel or mport,
1067	* -EINVAL - the channel is not in IDLE state,
1068	* -EAGAIN - no connection request available immediately (timeout=0),
1069	* -ENOMEM - unable to allocate new channel,
1070	* -ETIME - wait timeout expired,
1071	* -EINTR - wait was interrupted.
1072	*/
1073	static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
1074	long timeout)
1075	{
1076	struct rio_channel *ch;
1077	struct rio_channel *new_ch;
1078	struct conn_req *req;
1079	struct cm_peer *peer;
1080	int found = 0;
1081	int err = 0;
1082	long wret;
1083
1084	ch = riocm_get_channel(nr: ch_id);
1085	if (!ch)
1086	return ERR_PTR(error: -EINVAL);
1087
1088	if (!riocm_cmp(ch, cmp: RIO_CM_LISTEN)) {
1089	err = -EINVAL;
1090	goto err_put;
1091	}
1092
1093	/* Don't sleep if this is a non blocking call */
1094	if (!timeout) {
1095	if (!try_wait_for_completion(x: &ch->comp)) {
1096	err = -EAGAIN;
1097	goto err_put;
1098	}
1099	} else {
1100	riocm_debug(WAIT, "on %d", ch->id);
1101
1102	wret = wait_for_completion_interruptible_timeout(x: &ch->comp,
1103	timeout);
1104	if (!wret) {
1105	err = -ETIME;
1106	goto err_put;
1107	} else if (wret == -ERESTARTSYS) {
1108	err = -EINTR;
1109	goto err_put;
1110	}
1111	}
1112
1113	spin_lock_bh(lock: &ch->lock);
1114
1115	if (ch->state != RIO_CM_LISTEN) {
1116	err = -ECANCELED;
1117	} else if (list_empty(head: &ch->accept_queue)) {
1118	riocm_debug(WAIT, "on %d accept_queue is empty on completion",
1119	ch->id);
1120	err = -EIO;
1121	}
1122
1123	spin_unlock_bh(lock: &ch->lock);
1124
1125	if (err) {
1126	riocm_debug(WAIT, "on %d returns %d", ch->id, err);
1127	goto err_put;
1128	}
1129
1130	/* Create new channel for this connection */
1131	new_ch = riocm_ch_alloc(RIOCM_CHNUM_AUTO);
1132
1133	if (IS_ERR(ptr: new_ch)) {
1134	riocm_error("failed to get channel for new req (%ld)",
1135	PTR_ERR(new_ch));
1136	err = -ENOMEM;
1137	goto err_put;
1138	}
1139
1140	spin_lock_bh(lock: &ch->lock);
1141
1142	req = list_first_entry(&ch->accept_queue, struct conn_req, node);
1143	list_del(entry: &req->node);
1144	new_ch->cmdev = ch->cmdev;
1145	new_ch->loc_destid = ch->loc_destid;
1146	new_ch->rem_destid = req->destid;
1147	new_ch->rem_channel = req->chan;
1148
1149	spin_unlock_bh(lock: &ch->lock);
1150	riocm_put_channel(ch);
1151	ch = NULL;
1152	kfree(objp: req);
1153
1154	down_read(sem: &rdev_sem);
1155	/* Find requester's device object */
1156	list_for_each_entry(peer, &new_ch->cmdev->peers, node) {
1157	if (peer->rdev->destid == new_ch->rem_destid) {
1158	riocm_debug(RX_CMD, "found matching device(%s)",
1159	rio_name(peer->rdev));
1160	found = 1;
1161	break;
1162	}
1163	}
1164	up_read(sem: &rdev_sem);
1165
1166	if (!found) {
1167	/* If peer device object not found, simply ignore the request */
1168	err = -ENODEV;
1169	goto err_put_new_ch;
1170	}
1171
1172	new_ch->rdev = peer->rdev;
1173	new_ch->state = RIO_CM_CONNECTED;
1174	spin_lock_init(&new_ch->lock);
1175
1176	/* Acknowledge the connection request. */
1177	riocm_send_ack(ch: new_ch);
1178
1179	*new_ch_id = new_ch->id;
1180	return new_ch;
1181
1182	err_put_new_ch:
1183	spin_lock_bh(lock: &idr_lock);
1184	idr_remove(&ch_idr, id: new_ch->id);
1185	spin_unlock_bh(lock: &idr_lock);
1186	riocm_put_channel(ch: new_ch);
1187
1188	err_put:
1189	if (ch)
1190	riocm_put_channel(ch);
1191	*new_ch_id = 0;
1192	return ERR_PTR(error: err);
1193	}
1194
1195	/*
1196	* riocm_ch_listen - puts a channel into LISTEN state
1197	* @ch_id: channel ID
1198	*
1199	* Returns: 0 if success, or
1200	* -EINVAL if the specified channel does not exists or
1201	* is not in CHAN_BOUND state.
1202	*/
1203	static int riocm_ch_listen(u16 ch_id)
1204	{
1205	struct rio_channel *ch = NULL;
1206	int ret = 0;
1207
1208	riocm_debug(CHOP, "(ch_%d)", ch_id);
1209
1210	ch = riocm_get_channel(nr: ch_id);
1211	if (!ch)
1212	return -EINVAL;
1213	if (!riocm_cmp_exch(ch, cmp: RIO_CM_CHAN_BOUND, exch: RIO_CM_LISTEN))
1214	ret = -EINVAL;
1215	riocm_put_channel(ch);
1216	return ret;
1217	}
1218
1219	/*
1220	* riocm_ch_bind - associate a channel object and an mport device
1221	* @ch_id: channel ID
1222	* @mport_id: local mport device ID
1223	* @context: pointer to the additional caller's context
1224	*
1225	* Returns: 0 if success, or
1226	* -ENODEV if cannot find specified mport,
1227	* -EINVAL if the specified channel does not exist or
1228	* is not in IDLE state.
1229	*/
1230	static int riocm_ch_bind(u16 ch_id, u8 mport_id, void *context)
1231	{
1232	struct rio_channel *ch = NULL;
1233	struct cm_dev *cm;
1234	int rc = -ENODEV;
1235
1236	riocm_debug(CHOP, "ch_%d to mport_%d", ch_id, mport_id);
1237
1238	/* Find matching cm_dev object */
1239	down_read(sem: &rdev_sem);
1240	list_for_each_entry(cm, &cm_dev_list, list) {
1241	if ((cm->mport->id == mport_id) &&
1242	rio_mport_is_running(mport: cm->mport)) {
1243	rc = 0;
1244	break;
1245	}
1246	}
1247
1248	if (rc)
1249	goto exit;
1250
1251	ch = riocm_get_channel(nr: ch_id);
1252	if (!ch) {
1253	rc = -EINVAL;
1254	goto exit;
1255	}
1256
1257	spin_lock_bh(lock: &ch->lock);
1258	if (ch->state != RIO_CM_IDLE) {
1259	spin_unlock_bh(lock: &ch->lock);
1260	rc = -EINVAL;
1261	goto err_put;
1262	}
1263
1264	ch->cmdev = cm;
1265	ch->loc_destid = cm->mport->host_deviceid;
1266	ch->context = context;
1267	ch->state = RIO_CM_CHAN_BOUND;
1268	spin_unlock_bh(lock: &ch->lock);
1269	err_put:
1270	riocm_put_channel(ch);
1271	exit:
1272	up_read(sem: &rdev_sem);
1273	return rc;
1274	}
1275
1276	/*
1277	* riocm_ch_alloc - channel object allocation helper routine
1278	* @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1279	*
1280	* Return value: pointer to newly created channel object,
1281	* or error-valued pointer
1282	*/
1283	static struct rio_channel *riocm_ch_alloc(u16 ch_num)
1284	{
1285	int id;
1286	int start, end;
1287	struct rio_channel *ch;
1288
1289	ch = kzalloc(size: sizeof(*ch), GFP_KERNEL);
1290	if (!ch)
1291	return ERR_PTR(error: -ENOMEM);
1292
1293	if (ch_num) {
1294	/* If requested, try to obtain the specified channel ID */
1295	start = ch_num;
1296	end = ch_num + 1;
1297	} else {
1298	/* Obtain channel ID from the dynamic allocation range */
1299	start = chstart;
1300	end = RIOCM_MAX_CHNUM + 1;
1301	}
1302
1303	idr_preload(GFP_KERNEL);
1304	spin_lock_bh(lock: &idr_lock);
1305	id = idr_alloc_cyclic(&ch_idr, ptr: ch, start, end, GFP_NOWAIT);
1306	spin_unlock_bh(lock: &idr_lock);
1307	idr_preload_end();
1308
1309	if (id < 0) {
1310	kfree(objp: ch);
1311	return ERR_PTR(error: id == -ENOSPC ? -EBUSY : id);
1312	}
1313
1314	ch->id = (u16)id;
1315	ch->state = RIO_CM_IDLE;
1316	spin_lock_init(&ch->lock);
1317	INIT_LIST_HEAD(list: &ch->accept_queue);
1318	INIT_LIST_HEAD(list: &ch->ch_node);
1319	init_completion(x: &ch->comp);
1320	init_completion(x: &ch->comp_close);
1321	kref_init(kref: &ch->ref);
1322	ch->rx_ring.head = 0;
1323	ch->rx_ring.tail = 0;
1324	ch->rx_ring.count = 0;
1325	ch->rx_ring.inuse_cnt = 0;
1326
1327	return ch;
1328	}
1329
1330	/*
1331	* riocm_ch_create - creates a new channel object and allocates ID for it
1332	* @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1333	*
1334	* Allocates and initializes a new channel object. If the parameter ch_num > 0
1335	* and is within the valid range, riocm_ch_create tries to allocate the
1336	* specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1337	* automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1338	* Module parameter 'chstart' defines start of an ID range available for dynamic
1339	* allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1340	* Available channel numbers are limited by 16-bit size of channel numbers used
1341	* in the packet header.
1342	*
1343	* Return value: PTR to rio_channel structure if successful (with channel number
1344	* updated via pointer) or error-valued pointer if error.
1345	*/
1346	static struct rio_channel *riocm_ch_create(u16 *ch_num)
1347	{
1348	struct rio_channel *ch = NULL;
1349
1350	ch = riocm_ch_alloc(ch_num: *ch_num);
1351
1352	if (IS_ERR(ptr: ch))
1353	riocm_debug(CHOP, "Failed to allocate channel %d (err=%ld)",
1354	*ch_num, PTR_ERR(ch));
1355	else
1356	*ch_num = ch->id;
1357
1358	return ch;
1359	}
1360
1361	/*
1362	* riocm_ch_free - channel object release routine
1363	* @ref: pointer to a channel's kref structure
1364	*/
1365	static void riocm_ch_free(struct kref *ref)
1366	{
1367	struct rio_channel *ch = container_of(ref, struct rio_channel, ref);
1368	int i;
1369
1370	riocm_debug(CHOP, "(ch_%d)", ch->id);
1371
1372	if (ch->rx_ring.inuse_cnt) {
1373	for (i = 0;
1374	i < RIOCM_RX_RING_SIZE && ch->rx_ring.inuse_cnt; i++) {
1375	if (ch->rx_ring.inuse[i] != NULL) {
1376	kfree(objp: ch->rx_ring.inuse[i]);
1377	ch->rx_ring.inuse_cnt--;
1378	}
1379	}
1380	}
1381
1382	if (ch->rx_ring.count)
1383	for (i = 0; i < RIOCM_RX_RING_SIZE && ch->rx_ring.count; i++) {
1384	if (ch->rx_ring.buf[i] != NULL) {
1385	kfree(objp: ch->rx_ring.buf[i]);
1386	ch->rx_ring.count--;
1387	}
1388	}
1389
1390	complete(&ch->comp_close);
1391	}
1392
1393	static int riocm_send_close(struct rio_channel *ch)
1394	{
1395	struct rio_ch_chan_hdr *hdr;
1396	int ret;
1397
1398	/*
1399	* Send CH_CLOSE notification to the remote RapidIO device
1400	*/
1401
1402	hdr = kzalloc(size: sizeof(*hdr), GFP_KERNEL);
1403	if (hdr == NULL)
1404	return -ENOMEM;
1405
1406	hdr->bhdr.src_id = htonl(ch->loc_destid);
1407	hdr->bhdr.dst_id = htonl(ch->rem_destid);
1408	hdr->bhdr.src_mbox = cmbox;
1409	hdr->bhdr.dst_mbox = cmbox;
1410	hdr->bhdr.type = RIO_CM_CHAN;
1411	hdr->ch_op = CM_CONN_CLOSE;
1412	hdr->dst_ch = htons(ch->rem_channel);
1413	hdr->src_ch = htons(ch->id);
1414
1415	/* ATTN: the function call below relies on the fact that underlying
1416	* add_outb_message() routine copies TX data into its internal transfer
1417	* buffer. Needs to be reviewed if switched to direct buffer mode.
1418	*/
1419	ret = riocm_post_send(cm: ch->cmdev, rdev: ch->rdev, buffer: hdr, len: sizeof(*hdr));
1420
1421	if (ret == -EBUSY && !riocm_queue_req(cm: ch->cmdev, rdev: ch->rdev,
1422	buffer: hdr, len: sizeof(*hdr)))
1423	return 0;
1424	kfree(objp: hdr);
1425
1426	if (ret)
1427	riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch->id, ret);
1428
1429	return ret;
1430	}
1431
1432	/*
1433	* riocm_ch_close - closes a channel object with specified ID (by local request)
1434	* @ch: channel to be closed
1435	*/
1436	static int riocm_ch_close(struct rio_channel *ch)
1437	{
1438	unsigned long tmo = msecs_to_jiffies(m: 3000);
1439	enum rio_cm_state state;
1440	long wret;
1441	int ret = 0;
1442
1443	riocm_debug(CHOP, "ch_%d by %s(%d)",
1444	ch->id, current->comm, task_pid_nr(current));
1445
1446	state = riocm_exch(ch, exch: RIO_CM_DESTROYING);
1447	if (state == RIO_CM_CONNECTED)
1448	riocm_send_close(ch);
1449
1450	complete_all(&ch->comp);
1451
1452	riocm_put_channel(ch);
1453	wret = wait_for_completion_interruptible_timeout(x: &ch->comp_close, timeout: tmo);
1454
1455	riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1456
1457	if (wret == 0) {
1458	/* Timeout on wait occurred */
1459	riocm_debug(CHOP, "%s(%d) timed out waiting for ch %d",
1460	current->comm, task_pid_nr(current), ch->id);
1461	ret = -ETIMEDOUT;
1462	} else if (wret == -ERESTARTSYS) {
1463	/* Wait_for_completion was interrupted by a signal */
1464	riocm_debug(CHOP, "%s(%d) wait for ch %d was interrupted",
1465	current->comm, task_pid_nr(current), ch->id);
1466	ret = -EINTR;
1467	}
1468
1469	if (!ret) {
1470	riocm_debug(CHOP, "ch_%d resources released", ch->id);
1471	kfree(objp: ch);
1472	} else {
1473	riocm_debug(CHOP, "failed to release ch_%d resources", ch->id);
1474	}
1475
1476	return ret;
1477	}
1478
1479	/*
1480	* riocm_cdev_open() - Open character device
1481	*/
1482	static int riocm_cdev_open(struct inode *inode, struct file *filp)
1483	{
1484	riocm_debug(INIT, "by %s(%d) filp=%p ",
1485	current->comm, task_pid_nr(current), filp);
1486
1487	if (list_empty(head: &cm_dev_list))
1488	return -ENODEV;
1489
1490	return 0;
1491	}
1492
1493	/*
1494	* riocm_cdev_release() - Release character device
1495	*/
1496	static int riocm_cdev_release(struct inode *inode, struct file *filp)
1497	{
1498	struct rio_channel *ch, *_c;
1499	unsigned int i;
1500	LIST_HEAD(list);
1501
1502	riocm_debug(EXIT, "by %s(%d) filp=%p",
1503	current->comm, task_pid_nr(current), filp);
1504
1505	/* Check if there are channels associated with this file descriptor */
1506	spin_lock_bh(lock: &idr_lock);
1507	idr_for_each_entry(&ch_idr, ch, i) {
1508	if (ch && ch->filp == filp) {
1509	riocm_debug(EXIT, "ch_%d not released by %s(%d)",
1510	ch->id, current->comm,
1511	task_pid_nr(current));
1512	idr_remove(&ch_idr, id: ch->id);
1513	list_add(new: &ch->ch_node, head: &list);
1514	}
1515	}
1516	spin_unlock_bh(lock: &idr_lock);
1517
1518	if (!list_empty(head: &list)) {
1519	list_for_each_entry_safe(ch, _c, &list, ch_node) {
1520	list_del(entry: &ch->ch_node);
1521	riocm_ch_close(ch);
1522	}
1523	}
1524
1525	return 0;
1526	}
1527
1528	/*
1529	* cm_ep_get_list_size() - Reports number of endpoints in the network
1530	*/
1531	static int cm_ep_get_list_size(void __user *arg)
1532	{
1533	u32 __user *p = arg;
1534	u32 mport_id;
1535	u32 count = 0;
1536	struct cm_dev *cm;
1537
1538	if (get_user(mport_id, p))
1539	return -EFAULT;
1540	if (mport_id >= RIO_MAX_MPORTS)
1541	return -EINVAL;
1542
1543	/* Find a matching cm_dev object */
1544	down_read(sem: &rdev_sem);
1545	list_for_each_entry(cm, &cm_dev_list, list) {
1546	if (cm->mport->id == mport_id) {
1547	count = cm->npeers;
1548	up_read(sem: &rdev_sem);
1549	if (copy_to_user(to: arg, from: &count, n: sizeof(u32)))
1550	return -EFAULT;
1551	return 0;
1552	}
1553	}
1554	up_read(sem: &rdev_sem);
1555
1556	return -ENODEV;
1557	}
1558
1559	/*
1560	* cm_ep_get_list() - Returns list of attached endpoints
1561	*/
1562	static int cm_ep_get_list(void __user *arg)
1563	{
1564	struct cm_dev *cm;
1565	struct cm_peer *peer;
1566	u32 info[2];
1567	void *buf;
1568	u32 nent;
1569	u32 *entry_ptr;
1570	u32 i = 0;
1571	int ret = 0;
1572
1573	if (copy_from_user(to: &info, from: arg, n: sizeof(info)))
1574	return -EFAULT;
1575
1576	if (info[1] >= RIO_MAX_MPORTS || info[0] > RIOCM_MAX_EP_COUNT)
1577	return -EINVAL;
1578
1579	/* Find a matching cm_dev object */
1580	down_read(sem: &rdev_sem);
1581	list_for_each_entry(cm, &cm_dev_list, list)
1582	if (cm->mport->id == (u8)info[1])
1583	goto found;
1584
1585	up_read(sem: &rdev_sem);
1586	return -ENODEV;
1587
1588	found:
1589	nent = min(info[0], cm->npeers);
1590	buf = kcalloc(n: nent + 2, size: sizeof(u32), GFP_KERNEL);
1591	if (!buf) {
1592	up_read(sem: &rdev_sem);
1593	return -ENOMEM;
1594	}
1595
1596	entry_ptr = (u32 *)((uintptr_t)buf + 2*sizeof(u32));
1597
1598	list_for_each_entry(peer, &cm->peers, node) {
1599	*entry_ptr = (u32)peer->rdev->destid;
1600	entry_ptr++;
1601	if (++i == nent)
1602	break;
1603	}
1604	up_read(sem: &rdev_sem);
1605
1606	((u32 *)buf)[0] = i; /* report an updated number of entries */
1607	((u32 *)buf)[1] = info[1]; /* put back an mport ID */
1608	if (copy_to_user(to: arg, from: buf, n: sizeof(u32) * (info[0] + 2)))
1609	ret = -EFAULT;
1610
1611	kfree(objp: buf);
1612	return ret;
1613	}
1614
1615	/*
1616	* cm_mport_get_list() - Returns list of available local mport devices
1617	*/
1618	static int cm_mport_get_list(void __user *arg)
1619	{
1620	int ret = 0;
1621	u32 entries;
1622	void *buf;
1623	struct cm_dev *cm;
1624	u32 *entry_ptr;
1625	int count = 0;
1626
1627	if (copy_from_user(to: &entries, from: arg, n: sizeof(entries)))
1628	return -EFAULT;
1629	if (entries == 0 || entries > RIO_MAX_MPORTS)
1630	return -EINVAL;
1631	buf = kcalloc(n: entries + 1, size: sizeof(u32), GFP_KERNEL);
1632	if (!buf)
1633	return -ENOMEM;
1634
1635	/* Scan all registered cm_dev objects */
1636	entry_ptr = (u32 *)((uintptr_t)buf + sizeof(u32));
1637	down_read(sem: &rdev_sem);
1638	list_for_each_entry(cm, &cm_dev_list, list) {
1639	if (count++ < entries) {
1640	*entry_ptr = (cm->mport->id << 16) |
1641	cm->mport->host_deviceid;
1642	entry_ptr++;
1643	}
1644	}
1645	up_read(sem: &rdev_sem);
1646
1647	*((u32 *)buf) = count; /* report a real number of entries */
1648	if (copy_to_user(to: arg, from: buf, n: sizeof(u32) * (count + 1)))
1649	ret = -EFAULT;
1650
1651	kfree(objp: buf);
1652	return ret;
1653	}
1654
1655	/*
1656	* cm_chan_create() - Create a message exchange channel
1657	*/
1658	static int cm_chan_create(struct file *filp, void __user *arg)
1659	{
1660	u16 __user *p = arg;
1661	u16 ch_num;
1662	struct rio_channel *ch;
1663
1664	if (get_user(ch_num, p))
1665	return -EFAULT;
1666
1667	riocm_debug(CHOP, "ch_%d requested by %s(%d)",
1668	ch_num, current->comm, task_pid_nr(current));
1669	ch = riocm_ch_create(ch_num: &ch_num);
1670	if (IS_ERR(ptr: ch))
1671	return PTR_ERR(ptr: ch);
1672
1673	ch->filp = filp;
1674	riocm_debug(CHOP, "ch_%d created by %s(%d)",
1675	ch_num, current->comm, task_pid_nr(current));
1676	return put_user(ch_num, p);
1677	}
1678
1679	/*
1680	* cm_chan_close() - Close channel
1681	* @filp: Pointer to file object
1682	* @arg: Channel to close
1683	*/
1684	static int cm_chan_close(struct file *filp, void __user *arg)
1685	{
1686	u16 __user *p = arg;
1687	u16 ch_num;
1688	struct rio_channel *ch;
1689
1690	if (get_user(ch_num, p))
1691	return -EFAULT;
1692
1693	riocm_debug(CHOP, "ch_%d by %s(%d)",
1694	ch_num, current->comm, task_pid_nr(current));
1695
1696	spin_lock_bh(lock: &idr_lock);
1697	ch = idr_find(&ch_idr, id: ch_num);
1698	if (!ch) {
1699	spin_unlock_bh(lock: &idr_lock);
1700	return 0;
1701	}
1702	if (ch->filp != filp) {
1703	spin_unlock_bh(lock: &idr_lock);
1704	return -EINVAL;
1705	}
1706	idr_remove(&ch_idr, id: ch->id);
1707	spin_unlock_bh(lock: &idr_lock);
1708
1709	return riocm_ch_close(ch);
1710	}
1711
1712	/*
1713	* cm_chan_bind() - Bind channel
1714	* @arg: Channel number
1715	*/
1716	static int cm_chan_bind(void __user *arg)
1717	{
1718	struct rio_cm_channel chan;
1719
1720	if (copy_from_user(to: &chan, from: arg, n: sizeof(chan)))
1721	return -EFAULT;
1722	if (chan.mport_id >= RIO_MAX_MPORTS)
1723	return -EINVAL;
1724
1725	return riocm_ch_bind(ch_id: chan.id, mport_id: chan.mport_id, NULL);
1726	}
1727
1728	/*
1729	* cm_chan_listen() - Listen on channel
1730	* @arg: Channel number
1731	*/
1732	static int cm_chan_listen(void __user *arg)
1733	{
1734	u16 __user *p = arg;
1735	u16 ch_num;
1736
1737	if (get_user(ch_num, p))
1738	return -EFAULT;
1739
1740	return riocm_ch_listen(ch_id: ch_num);
1741	}
1742
1743	/*
1744	* cm_chan_accept() - Accept incoming connection
1745	* @filp: Pointer to file object
1746	* @arg: Channel number
1747	*/
1748	static int cm_chan_accept(struct file *filp, void __user *arg)
1749	{
1750	struct rio_cm_accept param;
1751	long accept_to;
1752	struct rio_channel *ch;
1753
1754	if (copy_from_user(to: &param, from: arg, n: sizeof(param)))
1755	return -EFAULT;
1756
1757	riocm_debug(CHOP, "on ch_%d by %s(%d)",
1758	param.ch_num, current->comm, task_pid_nr(current));
1759
1760	accept_to = param.wait_to ?
1761	msecs_to_jiffies(m: param.wait_to) : 0;
1762
1763	ch = riocm_ch_accept(ch_id: param.ch_num, new_ch_id: &param.ch_num, timeout: accept_to);
1764	if (IS_ERR(ptr: ch))
1765	return PTR_ERR(ptr: ch);
1766	ch->filp = filp;
1767
1768	riocm_debug(CHOP, "new ch_%d for %s(%d)",
1769	ch->id, current->comm, task_pid_nr(current));
1770
1771	if (copy_to_user(to: arg, from: &param, n: sizeof(param)))
1772	return -EFAULT;
1773	return 0;
1774	}
1775
1776	/*
1777	* cm_chan_connect() - Connect on channel
1778	* @arg: Channel information
1779	*/
1780	static int cm_chan_connect(void __user *arg)
1781	{
1782	struct rio_cm_channel chan;
1783	struct cm_dev *cm;
1784	struct cm_peer *peer;
1785	int ret = -ENODEV;
1786
1787	if (copy_from_user(to: &chan, from: arg, n: sizeof(chan)))
1788	return -EFAULT;
1789	if (chan.mport_id >= RIO_MAX_MPORTS)
1790	return -EINVAL;
1791
1792	down_read(sem: &rdev_sem);
1793
1794	/* Find matching cm_dev object */
1795	list_for_each_entry(cm, &cm_dev_list, list) {
1796	if (cm->mport->id == chan.mport_id) {
1797	ret = 0;
1798	break;
1799	}
1800	}
1801
1802	if (ret)
1803	goto err_out;
1804
1805	if (chan.remote_destid >= RIO_ANY_DESTID(cm->mport->sys_size)) {
1806	ret = -EINVAL;
1807	goto err_out;
1808	}
1809
1810	/* Find corresponding RapidIO endpoint device object */
1811	ret = -ENODEV;
1812
1813	list_for_each_entry(peer, &cm->peers, node) {
1814	if (peer->rdev->destid == chan.remote_destid) {
1815	ret = 0;
1816	break;
1817	}
1818	}
1819
1820	if (ret)
1821	goto err_out;
1822
1823	up_read(sem: &rdev_sem);
1824
1825	return riocm_ch_connect(loc_ch: chan.id, cm, peer, rem_ch: chan.remote_channel);
1826	err_out:
1827	up_read(sem: &rdev_sem);
1828	return ret;
1829	}
1830
1831	/*
1832	* cm_chan_msg_send() - Send a message through channel
1833	* @arg: Outbound message information
1834	*/
1835	static int cm_chan_msg_send(void __user *arg)
1836	{
1837	struct rio_cm_msg msg;
1838	void *buf;
1839	int ret;
1840
1841	if (copy_from_user(to: &msg, from: arg, n: sizeof(msg)))
1842	return -EFAULT;
1843	if (msg.size > RIO_MAX_MSG_SIZE)
1844	return -EINVAL;
1845
1846	buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
1847	if (IS_ERR(ptr: buf))
1848	return PTR_ERR(ptr: buf);
1849
1850	ret = riocm_ch_send(ch_id: msg.ch_num, buf, len: msg.size);
1851
1852	kfree(objp: buf);
1853	return ret;
1854	}
1855
1856	/*
1857	* cm_chan_msg_rcv() - Receive a message through channel
1858	* @arg: Inbound message information
1859	*/
1860	static int cm_chan_msg_rcv(void __user *arg)
1861	{
1862	struct rio_cm_msg msg;
1863	struct rio_channel *ch;
1864	void *buf;
1865	long rxto;
1866	int ret = 0, msg_size;
1867
1868	if (copy_from_user(to: &msg, from: arg, n: sizeof(msg)))
1869	return -EFAULT;
1870
1871	if (msg.ch_num == 0 || msg.size == 0)
1872	return -EINVAL;
1873
1874	ch = riocm_get_channel(nr: msg.ch_num);
1875	if (!ch)
1876	return -ENODEV;
1877
1878	rxto = msg.rxto ? msecs_to_jiffies(m: msg.rxto) : MAX_SCHEDULE_TIMEOUT;
1879
1880	ret = riocm_ch_receive(ch, buf: &buf, timeout: rxto);
1881	if (ret)
1882	goto out;
1883
1884	msg_size = min(msg.size, (u16)(RIO_MAX_MSG_SIZE));
1885
1886	if (copy_to_user(to: (void __user *)(uintptr_t)msg.msg, from: buf, n: msg_size))
1887	ret = -EFAULT;
1888
1889	riocm_ch_free_rxbuf(ch, buf);
1890	out:
1891	riocm_put_channel(ch);
1892	return ret;
1893	}
1894
1895	/*
1896	* riocm_cdev_ioctl() - IOCTL requests handler
1897	*/
1898	static long
1899	riocm_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1900	{
1901	switch (cmd) {
1902	case RIO_CM_EP_GET_LIST_SIZE:
1903	return cm_ep_get_list_size(arg: (void __user *)arg);
1904	case RIO_CM_EP_GET_LIST:
1905	return cm_ep_get_list(arg: (void __user *)arg);
1906	case RIO_CM_CHAN_CREATE:
1907	return cm_chan_create(filp, arg: (void __user *)arg);
1908	case RIO_CM_CHAN_CLOSE:
1909	return cm_chan_close(filp, arg: (void __user *)arg);
1910	case RIO_CM_CHAN_BIND:
1911	return cm_chan_bind(arg: (void __user *)arg);
1912	case RIO_CM_CHAN_LISTEN:
1913	return cm_chan_listen(arg: (void __user *)arg);
1914	case RIO_CM_CHAN_ACCEPT:
1915	return cm_chan_accept(filp, arg: (void __user *)arg);
1916	case RIO_CM_CHAN_CONNECT:
1917	return cm_chan_connect(arg: (void __user *)arg);
1918	case RIO_CM_CHAN_SEND:
1919	return cm_chan_msg_send(arg: (void __user *)arg);
1920	case RIO_CM_CHAN_RECEIVE:
1921	return cm_chan_msg_rcv(arg: (void __user *)arg);
1922	case RIO_CM_MPORT_GET_LIST:
1923	return cm_mport_get_list(arg: (void __user *)arg);
1924	default:
1925	break;
1926	}
1927
1928	return -EINVAL;
1929	}
1930
1931	static const struct file_operations riocm_cdev_fops = {
1932	.owner = THIS_MODULE,
1933	.open = riocm_cdev_open,
1934	.release = riocm_cdev_release,
1935	.unlocked_ioctl = riocm_cdev_ioctl,
1936	};
1937
1938	/*
1939	* riocm_add_dev - add new remote RapidIO device into channel management core
1940	* @dev: device object associated with RapidIO device
1941	* @sif: subsystem interface
1942	*
1943	* Adds the specified RapidIO device (if applicable) into peers list of
1944	* the corresponding channel management device (cm_dev).
1945	*/
1946	static int riocm_add_dev(struct device *dev, struct subsys_interface *sif)
1947	{
1948	struct cm_peer *peer;
1949	struct rio_dev *rdev = to_rio_dev(dev);
1950	struct cm_dev *cm;
1951
1952	/* Check if the remote device has capabilities required to support CM */
1953	if (!dev_cm_capable(rdev))
1954	return 0;
1955
1956	riocm_debug(RDEV, "(%s)", rio_name(rdev));
1957
1958	peer = kmalloc(size: sizeof(*peer), GFP_KERNEL);
1959	if (!peer)
1960	return -ENOMEM;
1961
1962	/* Find a corresponding cm_dev object */
1963	down_write(sem: &rdev_sem);
1964	list_for_each_entry(cm, &cm_dev_list, list) {
1965	if (cm->mport == rdev->net->hport)
1966	goto found;
1967	}
1968
1969	up_write(sem: &rdev_sem);
1970	kfree(objp: peer);
1971	return -ENODEV;
1972
1973	found:
1974	peer->rdev = rdev;
1975	list_add_tail(new: &peer->node, head: &cm->peers);
1976	cm->npeers++;
1977
1978	up_write(sem: &rdev_sem);
1979	return 0;
1980	}
1981
1982	/*
1983	* riocm_remove_dev - remove remote RapidIO device from channel management core
1984	* @dev: device object associated with RapidIO device
1985	* @sif: subsystem interface
1986	*
1987	* Removes the specified RapidIO device (if applicable) from peers list of
1988	* the corresponding channel management device (cm_dev).
1989	*/
1990	static void riocm_remove_dev(struct device *dev, struct subsys_interface *sif)
1991	{
1992	struct rio_dev *rdev = to_rio_dev(dev);
1993	struct cm_dev *cm;
1994	struct cm_peer *peer;
1995	struct rio_channel *ch, *_c;
1996	unsigned int i;
1997	bool found = false;
1998	LIST_HEAD(list);
1999
2000	/* Check if the remote device has capabilities required to support CM */
2001	if (!dev_cm_capable(rdev))
2002	return;
2003
2004	riocm_debug(RDEV, "(%s)", rio_name(rdev));
2005
2006	/* Find matching cm_dev object */
2007	down_write(sem: &rdev_sem);
2008	list_for_each_entry(cm, &cm_dev_list, list) {
2009	if (cm->mport == rdev->net->hport) {
2010	found = true;
2011	break;
2012	}
2013	}
2014
2015	if (!found) {
2016	up_write(sem: &rdev_sem);
2017	return;
2018	}
2019
2020	/* Remove remote device from the list of peers */
2021	found = false;
2022	list_for_each_entry(peer, &cm->peers, node) {
2023	if (peer->rdev == rdev) {
2024	riocm_debug(RDEV, "removing peer %s", rio_name(rdev));
2025	found = true;
2026	list_del(entry: &peer->node);
2027	cm->npeers--;
2028	kfree(objp: peer);
2029	break;
2030	}
2031	}
2032
2033	up_write(sem: &rdev_sem);
2034
2035	if (!found)
2036	return;
2037
2038	/*
2039	* Release channels associated with this peer
2040	*/
2041
2042	spin_lock_bh(lock: &idr_lock);
2043	idr_for_each_entry(&ch_idr, ch, i) {
2044	if (ch && ch->rdev == rdev) {
2045	if (atomic_read(v: &rdev->state) != RIO_DEVICE_SHUTDOWN)
2046	riocm_exch(ch, exch: RIO_CM_DISCONNECT);
2047	idr_remove(&ch_idr, id: ch->id);
2048	list_add(new: &ch->ch_node, head: &list);
2049	}
2050	}
2051	spin_unlock_bh(lock: &idr_lock);
2052
2053	if (!list_empty(head: &list)) {
2054	list_for_each_entry_safe(ch, _c, &list, ch_node) {
2055	list_del(entry: &ch->ch_node);
2056	riocm_ch_close(ch);
2057	}
2058	}
2059	}
2060
2061	/*
2062	* riocm_cdev_add() - Create rio_cm char device
2063	* @devno: device number assigned to device (MAJ + MIN)
2064	*/
2065	static int riocm_cdev_add(dev_t devno)
2066	{
2067	int ret;
2068
2069	cdev_init(&riocm_cdev.cdev, &riocm_cdev_fops);
2070	riocm_cdev.cdev.owner = THIS_MODULE;
2071	ret = cdev_add(&riocm_cdev.cdev, devno, 1);
2072	if (ret < 0) {
2073	riocm_error("Cannot register a device with error %d", ret);
2074	return ret;
2075	}
2076
2077	riocm_cdev.dev = device_create(cls: &dev_class, NULL, devt: devno, NULL, DEV_NAME);
2078	if (IS_ERR(ptr: riocm_cdev.dev)) {
2079	cdev_del(&riocm_cdev.cdev);
2080	return PTR_ERR(ptr: riocm_cdev.dev);
2081	}
2082
2083	riocm_debug(MPORT, "Added %s cdev(%d:%d)",
2084	DEV_NAME, MAJOR(devno), MINOR(devno));
2085
2086	return 0;
2087	}
2088
2089	/*
2090	* riocm_add_mport - add new local mport device into channel management core
2091	* @dev: device object associated with mport
2092	*
2093	* When a new mport device is added, CM immediately reserves inbound and
2094	* outbound RapidIO mailboxes that will be used.
2095	*/
2096	static int riocm_add_mport(struct device *dev)
2097	{
2098	int rc;
2099	int i;
2100	struct cm_dev *cm;
2101	struct rio_mport *mport = to_rio_mport(dev);
2102
2103	riocm_debug(MPORT, "add mport %s", mport->name);
2104
2105	cm = kzalloc(size: sizeof(*cm), GFP_KERNEL);
2106	if (!cm)
2107	return -ENOMEM;
2108
2109	cm->mport = mport;
2110
2111	rc = rio_request_outb_mbox(mport, cm, cmbox,
2112	RIOCM_TX_RING_SIZE, riocm_outb_msg_event);
2113	if (rc) {
2114	riocm_error("failed to allocate OBMBOX_%d on %s",
2115	cmbox, mport->name);
2116	kfree(objp: cm);
2117	return -ENODEV;
2118	}
2119
2120	rc = rio_request_inb_mbox(mport, cm, cmbox,
2121	RIOCM_RX_RING_SIZE, riocm_inb_msg_event);
2122	if (rc) {
2123	riocm_error("failed to allocate IBMBOX_%d on %s",
2124	cmbox, mport->name);
2125	rio_release_outb_mbox(mport, cmbox);
2126	kfree(objp: cm);
2127	return -ENODEV;
2128	}
2129
2130	cm->rx_wq = create_workqueue(DRV_NAME "/rxq");
2131	if (!cm->rx_wq) {
2132	rio_release_inb_mbox(mport, cmbox);
2133	rio_release_outb_mbox(mport, cmbox);
2134	kfree(objp: cm);
2135	return -ENOMEM;
2136	}
2137
2138	/*
2139	* Allocate and register inbound messaging buffers to be ready
2140	* to receive channel and system management requests
2141	*/
2142	for (i = 0; i < RIOCM_RX_RING_SIZE; i++)
2143	cm->rx_buf[i] = NULL;
2144
2145	cm->rx_slots = RIOCM_RX_RING_SIZE;
2146	mutex_init(&cm->rx_lock);
2147	riocm_rx_fill(cm, RIOCM_RX_RING_SIZE);
2148	INIT_WORK(&cm->rx_work, rio_ibmsg_handler);
2149
2150	cm->tx_slot = 0;
2151	cm->tx_cnt = 0;
2152	cm->tx_ack_slot = 0;
2153	spin_lock_init(&cm->tx_lock);
2154
2155	INIT_LIST_HEAD(list: &cm->peers);
2156	cm->npeers = 0;
2157	INIT_LIST_HEAD(list: &cm->tx_reqs);
2158
2159	down_write(sem: &rdev_sem);
2160	list_add_tail(new: &cm->list, head: &cm_dev_list);
2161	up_write(sem: &rdev_sem);
2162
2163	return 0;
2164	}
2165
2166	/*
2167	* riocm_remove_mport - remove local mport device from channel management core
2168	* @dev: device object associated with mport
2169	*
2170	* Removes a local mport device from the list of registered devices that provide
2171	* channel management services. Returns an error if the specified mport is not
2172	* registered with the CM core.
2173	*/
2174	static void riocm_remove_mport(struct device *dev)
2175	{
2176	struct rio_mport *mport = to_rio_mport(dev);
2177	struct cm_dev *cm;
2178	struct cm_peer *peer, *temp;
2179	struct rio_channel *ch, *_c;
2180	unsigned int i;
2181	bool found = false;
2182	LIST_HEAD(list);
2183
2184	riocm_debug(MPORT, "%s", mport->name);
2185
2186	/* Find a matching cm_dev object */
2187	down_write(sem: &rdev_sem);
2188	list_for_each_entry(cm, &cm_dev_list, list) {
2189	if (cm->mport == mport) {
2190	list_del(entry: &cm->list);
2191	found = true;
2192	break;
2193	}
2194	}
2195	up_write(sem: &rdev_sem);
2196	if (!found)
2197	return;
2198
2199	flush_workqueue(cm->rx_wq);
2200	destroy_workqueue(wq: cm->rx_wq);
2201
2202	/* Release channels bound to this mport */
2203	spin_lock_bh(lock: &idr_lock);
2204	idr_for_each_entry(&ch_idr, ch, i) {
2205	if (ch->cmdev == cm) {
2206	riocm_debug(RDEV, "%s drop ch_%d",
2207	mport->name, ch->id);
2208	idr_remove(&ch_idr, id: ch->id);
2209	list_add(new: &ch->ch_node, head: &list);
2210	}
2211	}
2212	spin_unlock_bh(lock: &idr_lock);
2213
2214	if (!list_empty(head: &list)) {
2215	list_for_each_entry_safe(ch, _c, &list, ch_node) {
2216	list_del(entry: &ch->ch_node);
2217	riocm_ch_close(ch);
2218	}
2219	}
2220
2221	rio_release_inb_mbox(mport, cmbox);
2222	rio_release_outb_mbox(mport, cmbox);
2223
2224	/* Remove and free peer entries */
2225	if (!list_empty(head: &cm->peers))
2226	riocm_debug(RDEV, "ATTN: peer list not empty");
2227	list_for_each_entry_safe(peer, temp, &cm->peers, node) {
2228	riocm_debug(RDEV, "removing peer %s", rio_name(peer->rdev));
2229	list_del(entry: &peer->node);
2230	kfree(objp: peer);
2231	}
2232
2233	riocm_rx_free(cm);
2234	kfree(objp: cm);
2235	riocm_debug(MPORT, "%s done", mport->name);
2236	}
2237
2238	static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
2239	void *unused)
2240	{
2241	struct rio_channel *ch;
2242	unsigned int i;
2243	LIST_HEAD(list);
2244
2245	riocm_debug(EXIT, ".");
2246
2247	/*
2248	* If there are any channels left in connected state send
2249	* close notification to the connection partner.
2250	* First build a list of channels that require a closing
2251	* notification because function riocm_send_close() should
2252	* be called outside of spinlock protected code.
2253	*/
2254	spin_lock_bh(lock: &idr_lock);
2255	idr_for_each_entry(&ch_idr, ch, i) {
2256	if (ch->state == RIO_CM_CONNECTED) {
2257	riocm_debug(EXIT, "close ch %d", ch->id);
2258	idr_remove(&ch_idr, id: ch->id);
2259	list_add(new: &ch->ch_node, head: &list);
2260	}
2261	}
2262	spin_unlock_bh(lock: &idr_lock);
2263
2264	list_for_each_entry(ch, &list, ch_node)
2265	riocm_send_close(ch);
2266
2267	return NOTIFY_DONE;
2268	}
2269
2270	/*
2271	* riocm_interface handles addition/removal of remote RapidIO devices
2272	*/
2273	static struct subsys_interface riocm_interface = {
2274	.name = "rio_cm",
2275	.subsys = &rio_bus_type,
2276	.add_dev = riocm_add_dev,
2277	.remove_dev = riocm_remove_dev,
2278	};
2279
2280	/*
2281	* rio_mport_interface handles addition/removal local mport devices
2282	*/
2283	static struct class_interface rio_mport_interface __refdata = {
2284	.class = &rio_mport_class,
2285	.add_dev = riocm_add_mport,
2286	.remove_dev = riocm_remove_mport,
2287	};
2288
2289	static struct notifier_block rio_cm_notifier = {
2290	.notifier_call = rio_cm_shutdown,
2291	};
2292
2293	static int __init riocm_init(void)
2294	{
2295	int ret;
2296
2297	/* Create device class needed by udev */
2298	ret = class_register(class: &dev_class);
2299	if (ret) {
2300	riocm_error("Cannot create " DRV_NAME " class");
2301	return ret;
2302	}
2303
2304	ret = alloc_chrdev_region(&dev_number, 0, 1, DRV_NAME);
2305	if (ret) {
2306	class_unregister(class: &dev_class);
2307	return ret;
2308	}
2309
2310	dev_major = MAJOR(dev_number);
2311	dev_minor_base = MINOR(dev_number);
2312	riocm_debug(INIT, "Registered class with %d major", dev_major);
2313
2314	/*
2315	* Register as rapidio_port class interface to get notifications about
2316	* mport additions and removals.
2317	*/
2318	ret = class_interface_register(&rio_mport_interface);
2319	if (ret) {
2320	riocm_error("class_interface_register error: %d", ret);
2321	goto err_reg;
2322	}
2323
2324	/*
2325	* Register as RapidIO bus interface to get notifications about
2326	* addition/removal of remote RapidIO devices.
2327	*/
2328	ret = subsys_interface_register(sif: &riocm_interface);
2329	if (ret) {
2330	riocm_error("subsys_interface_register error: %d", ret);
2331	goto err_cl;
2332	}
2333
2334	ret = register_reboot_notifier(&rio_cm_notifier);
2335	if (ret) {
2336	riocm_error("failed to register reboot notifier (err=%d)", ret);
2337	goto err_sif;
2338	}
2339
2340	ret = riocm_cdev_add(devno: dev_number);
2341	if (ret) {
2342	unregister_reboot_notifier(&rio_cm_notifier);
2343	ret = -ENODEV;
2344	goto err_sif;
2345	}
2346
2347	return 0;
2348	err_sif:
2349	subsys_interface_unregister(sif: &riocm_interface);
2350	err_cl:
2351	class_interface_unregister(&rio_mport_interface);
2352	err_reg:
2353	unregister_chrdev_region(dev_number, 1);
2354	class_unregister(class: &dev_class);
2355	return ret;
2356	}
2357
2358	static void __exit riocm_exit(void)
2359	{
2360	riocm_debug(EXIT, "enter");
2361	unregister_reboot_notifier(&rio_cm_notifier);
2362	subsys_interface_unregister(sif: &riocm_interface);
2363	class_interface_unregister(&rio_mport_interface);
2364	idr_destroy(&ch_idr);
2365
2366	device_unregister(dev: riocm_cdev.dev);
2367	cdev_del(&(riocm_cdev.cdev));
2368
2369	class_unregister(class: &dev_class);
2370	unregister_chrdev_region(dev_number, 1);
2371	}
2372
2373	late_initcall(riocm_init);
2374	module_exit(riocm_exit);
2375