1	// SPDX-License-Identifier: GPL-2.0-only
2
3	/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
4	/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
5
6	#include <asm/byteorder.h>
7	#include <linux/completion.h>
8	#include <linux/crc32.h>
9	#include <linux/delay.h>
10	#include <linux/dma-mapping.h>
11	#include <linux/kref.h>
12	#include <linux/list.h>
13	#include <linux/mhi.h>
14	#include <linux/mm.h>
15	#include <linux/moduleparam.h>
16	#include <linux/mutex.h>
17	#include <linux/overflow.h>
18	#include <linux/pci.h>
19	#include <linux/scatterlist.h>
20	#include <linux/sched/signal.h>
21	#include <linux/types.h>
22	#include <linux/uaccess.h>
23	#include <linux/workqueue.h>
24	#include <linux/wait.h>
25	#include <drm/drm_device.h>
26	#include <drm/drm_file.h>
27	#include <uapi/drm/qaic_accel.h>
28
29	#include "qaic.h"
30
31	#define MANAGE_MAGIC_NUMBER ((__force __le32)0x43494151) /* "QAIC" in little endian */
32	#define QAIC_DBC_Q_GAP SZ_256
33	#define QAIC_DBC_Q_BUF_ALIGN SZ_4K
34	#define QAIC_MANAGE_WIRE_MSG_LENGTH SZ_64K /* Max DMA message length */
35	#define QAIC_WRAPPER_MAX_SIZE SZ_4K
36	#define QAIC_MHI_RETRY_WAIT_MS 100
37	#define QAIC_MHI_RETRY_MAX 20
38
39	static unsigned int control_resp_timeout_s = 60; /* 60 sec default */
40	module_param(control_resp_timeout_s, uint, 0600);
41	MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");
42
43	struct manage_msg {
44	u32 len;
45	u32 count;
46	u8 data[];
47	};
48
49	/*
50	* wire encoding structures for the manage protocol.
51	* All fields are little endian on the wire
52	*/
53	struct wire_msg_hdr {
54	__le32 crc32; /* crc of everything following this field in the message */
55	__le32 magic_number;
56	__le32 sequence_number;
57	__le32 len; /* length of this message */
58	__le32 count; /* number of transactions in this message */
59	__le32 handle; /* unique id to track the resources consumed */
60	__le32 partition_id; /* partition id for the request (signed) */
61	__le32 padding; /* must be 0 */
62	} __packed;
63
64	struct wire_msg {
65	struct wire_msg_hdr hdr;
66	u8 data[];
67	} __packed;
68
69	struct wire_trans_hdr {
70	__le32 type;
71	__le32 len;
72	} __packed;
73
74	/* Each message sent from driver to device are organized in a list of wrapper_msg */
75	struct wrapper_msg {
76	struct list_head list;
77	struct kref ref_count;
78	u32 len; /* length of data to transfer */
79	struct wrapper_list *head;
80	union {
81	struct wire_msg msg;
82	struct wire_trans_hdr trans;
83	};
84	};
85
86	struct wrapper_list {
87	struct list_head list;
88	spinlock_t lock; /* Protects the list state during additions and removals */
89	};
90
91	struct wire_trans_passthrough {
92	struct wire_trans_hdr hdr;
93	u8 data[];
94	} __packed;
95
96	struct wire_addr_size_pair {
97	__le64 addr;
98	__le64 size;
99	} __packed;
100
101	struct wire_trans_dma_xfer {
102	struct wire_trans_hdr hdr;
103	__le32 tag;
104	__le32 count;
105	__le32 dma_chunk_id;
106	__le32 padding;
107	struct wire_addr_size_pair data[];
108	} __packed;
109
110	/* Initiated by device to continue the DMA xfer of a large piece of data */
111	struct wire_trans_dma_xfer_cont {
112	struct wire_trans_hdr hdr;
113	__le32 dma_chunk_id;
114	__le32 padding;
115	__le64 xferred_size;
116	} __packed;
117
118	struct wire_trans_activate_to_dev {
119	struct wire_trans_hdr hdr;
120	__le64 req_q_addr;
121	__le64 rsp_q_addr;
122	__le32 req_q_size;
123	__le32 rsp_q_size;
124	__le32 buf_len;
125	__le32 options; /* unused, but BIT(16) has meaning to the device */
126	} __packed;
127
128	struct wire_trans_activate_from_dev {
129	struct wire_trans_hdr hdr;
130	__le32 status;
131	__le32 dbc_id;
132	__le64 options; /* unused */
133	} __packed;
134
135	struct wire_trans_deactivate_from_dev {
136	struct wire_trans_hdr hdr;
137	__le32 status;
138	__le32 dbc_id;
139	} __packed;
140
141	struct wire_trans_terminate_to_dev {
142	struct wire_trans_hdr hdr;
143	__le32 handle;
144	__le32 padding;
145	} __packed;
146
147	struct wire_trans_terminate_from_dev {
148	struct wire_trans_hdr hdr;
149	__le32 status;
150	__le32 padding;
151	} __packed;
152
153	struct wire_trans_status_to_dev {
154	struct wire_trans_hdr hdr;
155	} __packed;
156
157	struct wire_trans_status_from_dev {
158	struct wire_trans_hdr hdr;
159	__le16 major;
160	__le16 minor;
161	__le32 status;
162	__le64 status_flags;
163	} __packed;
164
165	struct wire_trans_validate_part_to_dev {
166	struct wire_trans_hdr hdr;
167	__le32 part_id;
168	__le32 padding;
169	} __packed;
170
171	struct wire_trans_validate_part_from_dev {
172	struct wire_trans_hdr hdr;
173	__le32 status;
174	__le32 padding;
175	} __packed;
176
177	struct xfer_queue_elem {
178	/*
179	* Node in list of ongoing transfer request on control channel.
180	* Maintained by root device struct.
181	*/
182	struct list_head list;
183	/* Sequence number of this transfer request */
184	u32 seq_num;
185	/* This is used to wait on until completion of transfer request */
186	struct completion xfer_done;
187	/* Received data from device */
188	void *buf;
189	};
190
191	struct dma_xfer {
192	/* Node in list of DMA transfers which is used for cleanup */
193	struct list_head list;
194	/* SG table of memory used for DMA */
195	struct sg_table *sgt;
196	/* Array pages used for DMA */
197	struct page **page_list;
198	/* Number of pages used for DMA */
199	unsigned long nr_pages;
200	};
201
202	struct ioctl_resources {
203	/* List of all DMA transfers which is used later for cleanup */
204	struct list_head dma_xfers;
205	/* Base address of request queue which belongs to a DBC */
206	void *buf;
207	/*
208	* Base bus address of request queue which belongs to a DBC. Response
209	* queue base bus address can be calculated by adding size of request
210	* queue to base bus address of request queue.
211	*/
212	dma_addr_t dma_addr;
213	/* Total size of request queue and response queue in byte */
214	u32 total_size;
215	/* Total number of elements that can be queued in each of request and response queue */
216	u32 nelem;
217	/* Base address of response queue which belongs to a DBC */
218	void *rsp_q_base;
219	/* Status of the NNC message received */
220	u32 status;
221	/* DBC id of the DBC received from device */
222	u32 dbc_id;
223	/*
224	* DMA transfer request messages can be big in size and it may not be
225	* possible to send them in one shot. In such cases the messages are
226	* broken into chunks, this field stores ID of such chunks.
227	*/
228	u32 dma_chunk_id;
229	/* Total number of bytes transferred for a DMA xfer request */
230	u64 xferred_dma_size;
231	/* Header of transaction message received from user. Used during DMA xfer request. */
232	void *trans_hdr;
233	};
234
235	struct resp_work {
236	struct work_struct work;
237	struct qaic_device *qdev;
238	void *buf;
239	};
240
241	/*
242	* Since we're working with little endian messages, its useful to be able to
243	* increment without filling a whole line with conversions back and forth just
244	* to add one(1) to a message count.
245	*/
246	static __le32 incr_le32(__le32 val)
247	{
248	return cpu_to_le32(le32_to_cpu(val) + 1);
249	}
250
251	static u32 gen_crc(void *msg)
252	{
253	struct wrapper_list *wrappers = msg;
254	struct wrapper_msg *w;
255	u32 crc = ~0;
256
257	list_for_each_entry(w, &wrappers->list, list)
258	crc = crc32(crc, p: &w->msg, len: w->len);
259
260	return crc ^ ~0;
261	}
262
263	static u32 gen_crc_stub(void *msg)
264	{
265	return 0;
266	}
267
268	static bool valid_crc(void *msg)
269	{
270	struct wire_msg_hdr *hdr = msg;
271	bool ret;
272	u32 crc;
273
274	/*
275	* The output of this algorithm is always converted to the native
276	* endianness.
277	*/
278	crc = le32_to_cpu(hdr->crc32);
279	hdr->crc32 = 0;
280	ret = (crc32(crc: ~0, p: msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;
281	hdr->crc32 = cpu_to_le32(crc);
282	return ret;
283	}
284
285	static bool valid_crc_stub(void *msg)
286	{
287	return true;
288	}
289
290	static void free_wrapper(struct kref *ref)
291	{
292	struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);
293
294	list_del(entry: &wrapper->list);
295	kfree(objp: wrapper);
296	}
297
298	static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,
299	struct qaic_user *usr)
300	{
301	u32 dbc_id = resources->dbc_id;
302
303	if (resources->buf) {
304	wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);
305	qdev->dbc[dbc_id].req_q_base = resources->buf;
306	qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;
307	qdev->dbc[dbc_id].dma_addr = resources->dma_addr;
308	qdev->dbc[dbc_id].total_size = resources->total_size;
309	qdev->dbc[dbc_id].nelem = resources->nelem;
310	enable_dbc(qdev, dbc_id, usr);
311	qdev->dbc[dbc_id].in_use = true;
312	resources->buf = NULL;
313	set_dbc_state(qdev, dbc_id, state: DBC_STATE_ASSIGNED);
314	}
315	}
316
317	static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)
318	{
319	if (resources->buf)
320	dma_free_coherent(dev: &qdev->pdev->dev, size: resources->total_size, cpu_addr: resources->buf,
321	dma_handle: resources->dma_addr);
322	resources->buf = NULL;
323	}
324
325	static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)
326	{
327	struct dma_xfer *xfer;
328	struct dma_xfer *x;
329	int i;
330
331	list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {
332	dma_unmap_sgtable(dev: &qdev->pdev->dev, sgt: xfer->sgt, dir: DMA_TO_DEVICE, attrs: 0);
333	sg_free_table(xfer->sgt);
334	kfree(objp: xfer->sgt);
335	for (i = 0; i < xfer->nr_pages; ++i)
336	put_page(page: xfer->page_list[i]);
337	kfree(objp: xfer->page_list);
338	list_del(entry: &xfer->list);
339	kfree(objp: xfer);
340	}
341	}
342
343	static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)
344	{
345	struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);
346
347	if (!w)
348	return NULL;
349	list_add_tail(new: &w->list, head: &wrappers->list);
350	kref_init(kref: &w->ref_count);
351	w->head = wrappers;
352	return w;
353	}
354
355	static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
356	u32 *user_len)
357	{
358	struct qaic_manage_trans_passthrough *in_trans = trans;
359	struct wire_trans_passthrough *out_trans;
360	struct wrapper_msg *trans_wrapper;
361	struct wrapper_msg *wrapper;
362	struct wire_msg *msg;
363	u32 msg_hdr_len;
364
365	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
366	msg = &wrapper->msg;
367	msg_hdr_len = le32_to_cpu(msg->hdr.len);
368
369	if (in_trans->hdr.len % 8 != 0)
370	return -EINVAL;
371
372	if (size_add(addend1: msg_hdr_len, addend2: in_trans->hdr.len) > QAIC_MANAGE_WIRE_MSG_LENGTH)
373	return -ENOSPC;
374
375	trans_wrapper = add_wrapper(wrappers,
376	offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);
377	if (!trans_wrapper)
378	return -ENOMEM;
379	trans_wrapper->len = in_trans->hdr.len;
380	out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;
381
382	memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));
383	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
384	msg->hdr.count = incr_le32(val: msg->hdr.count);
385	*user_len += in_trans->hdr.len;
386	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);
387	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
388
389	return 0;
390	}
391
392	/* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */
393	static int find_and_map_user_pages(struct qaic_device *qdev,
394	struct qaic_manage_trans_dma_xfer *in_trans,
395	struct ioctl_resources *resources, struct dma_xfer *xfer)
396	{
397	u64 xfer_start_addr, remaining, end, total;
398	unsigned long need_pages;
399	struct page **page_list;
400	unsigned long nr_pages;
401	struct sg_table *sgt;
402	int ret;
403	int i;
404
405	if (check_add_overflow(in_trans->addr, resources->xferred_dma_size, &xfer_start_addr))
406	return -EINVAL;
407
408	if (in_trans->size < resources->xferred_dma_size)
409	return -EINVAL;
410	remaining = in_trans->size - resources->xferred_dma_size;
411	if (remaining == 0)
412	return -EINVAL;
413
414	if (check_add_overflow(xfer_start_addr, remaining, &end))
415	return -EINVAL;
416
417	total = remaining + offset_in_page(xfer_start_addr);
418	if (total >= SIZE_MAX)
419	return -EINVAL;
420
421	need_pages = DIV_ROUND_UP(total, PAGE_SIZE);
422
423	nr_pages = need_pages;
424
425	while (1) {
426	page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);
427	if (!page_list) {
428	nr_pages = nr_pages / 2;
429	if (!nr_pages)
430	return -ENOMEM;
431	} else {
432	break;
433	}
434	}
435
436	ret = get_user_pages_fast(start: xfer_start_addr, nr_pages, gup_flags: 0, pages: page_list);
437	if (ret < 0)
438	goto free_page_list;
439	if (ret != nr_pages) {
440	nr_pages = ret;
441	ret = -EFAULT;
442	goto put_pages;
443	}
444
445	sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
446	if (!sgt) {
447	ret = -ENOMEM;
448	goto put_pages;
449	}
450
451	ret = sg_alloc_table_from_pages(sgt, pages: page_list, n_pages: nr_pages,
452	offset_in_page(xfer_start_addr),
453	size: remaining, GFP_KERNEL);
454	if (ret) {
455	ret = -ENOMEM;
456	goto free_sgt;
457	}
458
459	ret = dma_map_sgtable(dev: &qdev->pdev->dev, sgt, dir: DMA_TO_DEVICE, attrs: 0);
460	if (ret)
461	goto free_table;
462
463	xfer->sgt = sgt;
464	xfer->page_list = page_list;
465	xfer->nr_pages = nr_pages;
466
467	return need_pages > nr_pages ? 1 : 0;
468
469	free_table:
470	sg_free_table(sgt);
471	free_sgt:
472	kfree(objp: sgt);
473	put_pages:
474	for (i = 0; i < nr_pages; ++i)
475	put_page(page: page_list[i]);
476	free_page_list:
477	kfree(objp: page_list);
478	return ret;
479	}
480
481	/* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */
482	static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,
483	struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,
484	struct wire_trans_dma_xfer **out_trans)
485	{
486	struct wrapper_msg *trans_wrapper;
487	struct sg_table *sgt = xfer->sgt;
488	struct wire_addr_size_pair *asp;
489	struct scatterlist *sg;
490	struct wrapper_msg *w;
491	unsigned int dma_len;
492	u64 dma_chunk_len;
493	void *boundary;
494	int nents_dma;
495	int nents;
496	int i;
497
498	nents = sgt->nents;
499	nents_dma = nents;
500	*size = QAIC_MANAGE_WIRE_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);
501	for_each_sgtable_dma_sg(sgt, sg, i) {
502	*size -= sizeof(*asp);
503	/* Save 1K for possible follow-up transactions. */
504	if (*size < SZ_1K) {
505	nents_dma = i;
506	break;
507	}
508	}
509
510	trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
511	if (!trans_wrapper)
512	return -ENOMEM;
513	*out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;
514
515	asp = (*out_trans)->data;
516	boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;
517	*size = 0;
518
519	dma_len = 0;
520	w = trans_wrapper;
521	dma_chunk_len = 0;
522	for_each_sg(sgt->sgl, sg, nents_dma, i) {
523	asp->size = cpu_to_le64(dma_len);
524	dma_chunk_len += dma_len;
525	if (dma_len) {
526	asp++;
527	if ((void *)asp + sizeof(*asp) > boundary) {
528	w->len = (void *)asp - (void *)&w->msg;
529	*size += w->len;
530	w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);
531	if (!w)
532	return -ENOMEM;
533	boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;
534	asp = (struct wire_addr_size_pair *)&w->msg;
535	}
536	}
537	asp->addr = cpu_to_le64(sg_dma_address(sg));
538	dma_len = sg_dma_len(sg);
539	}
540	/* finalize the last segment */
541	asp->size = cpu_to_le64(dma_len);
542	w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;
543	*size += w->len;
544	dma_chunk_len += dma_len;
545	resources->xferred_dma_size += dma_chunk_len;
546
547	return nents_dma < nents ? 1 : 0;
548	}
549
550	static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)
551	{
552	int i;
553
554	dma_unmap_sgtable(dev: &qdev->pdev->dev, sgt: xfer->sgt, dir: DMA_TO_DEVICE, attrs: 0);
555	sg_free_table(xfer->sgt);
556	kfree(objp: xfer->sgt);
557	for (i = 0; i < xfer->nr_pages; ++i)
558	put_page(page: xfer->page_list[i]);
559	kfree(objp: xfer->page_list);
560	}
561
562	static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
563	u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)
564	{
565	struct qaic_manage_trans_dma_xfer *in_trans = trans;
566	struct wire_trans_dma_xfer *out_trans;
567	struct wrapper_msg *wrapper;
568	struct dma_xfer *xfer;
569	struct wire_msg *msg;
570	bool need_cont_dma;
571	u32 msg_hdr_len;
572	u32 size;
573	int ret;
574
575	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
576	msg = &wrapper->msg;
577	msg_hdr_len = le32_to_cpu(msg->hdr.len);
578
579	/* There should be enough space to hold at least one ASP entry. */
580	if (size_add(addend1: msg_hdr_len, addend2: sizeof(*out_trans) + sizeof(struct wire_addr_size_pair)) >
581	QAIC_MANAGE_WIRE_MSG_LENGTH)
582	return -ENOMEM;
583
584	xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);
585	if (!xfer)
586	return -ENOMEM;
587
588	ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);
589	if (ret < 0)
590	goto free_xfer;
591
592	need_cont_dma = (bool)ret;
593
594	ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, size: &size, out_trans: &out_trans);
595	if (ret < 0)
596	goto cleanup_xfer;
597
598	need_cont_dma = need_cont_dma || (bool)ret;
599
600	msg->hdr.len = cpu_to_le32(msg_hdr_len + size);
601	msg->hdr.count = incr_le32(val: msg->hdr.count);
602
603	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
604	out_trans->hdr.len = cpu_to_le32(size);
605	out_trans->tag = cpu_to_le32(in_trans->tag);
606	out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /
607	sizeof(struct wire_addr_size_pair));
608
609	*user_len += in_trans->hdr.len;
610
611	if (resources->dma_chunk_id) {
612	out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
613	} else if (need_cont_dma) {
614	while (resources->dma_chunk_id == 0)
615	resources->dma_chunk_id = atomic_inc_return(v: &usr->chunk_id);
616
617	out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);
618	}
619	resources->trans_hdr = trans;
620
621	list_add(new: &xfer->list, head: &resources->dma_xfers);
622	return 0;
623
624	cleanup_xfer:
625	cleanup_xfer(qdev, xfer);
626	free_xfer:
627	kfree(objp: xfer);
628	return ret;
629	}
630
631	static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
632	u32 *user_len, struct ioctl_resources *resources)
633	{
634	struct qaic_manage_trans_activate_to_dev *in_trans = trans;
635	struct wire_trans_activate_to_dev *out_trans;
636	struct wrapper_msg *trans_wrapper;
637	struct wrapper_msg *wrapper;
638	struct wire_msg *msg;
639	dma_addr_t dma_addr;
640	u32 msg_hdr_len;
641	void *buf;
642	u32 nelem;
643	u32 size;
644	int ret;
645
646	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
647	msg = &wrapper->msg;
648	msg_hdr_len = le32_to_cpu(msg->hdr.len);
649
650	if (size_add(addend1: msg_hdr_len, addend2: sizeof(*out_trans)) > QAIC_MANAGE_WIRE_MSG_LENGTH)
651	return -ENOSPC;
652
653	if (!in_trans->queue_size)
654	return -EINVAL;
655
656	if (in_trans->pad)
657	return -EINVAL;
658
659	nelem = in_trans->queue_size;
660	if (check_mul_overflow((u32)(get_dbc_req_elem_size() + get_dbc_rsp_elem_size()),
661	nelem,
662	&size))
663	return -EINVAL;
664
665	if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)
666	return -EINVAL;
667
668	size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);
669
670	buf = dma_alloc_coherent(dev: &qdev->pdev->dev, size, dma_handle: &dma_addr, GFP_KERNEL);
671	if (!buf)
672	return -ENOMEM;
673
674	trans_wrapper = add_wrapper(wrappers,
675	offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));
676	if (!trans_wrapper) {
677	ret = -ENOMEM;
678	goto free_dma;
679	}
680	trans_wrapper->len = sizeof(*out_trans);
681	out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;
682
683	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);
684	out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));
685	out_trans->buf_len = cpu_to_le32(size);
686	out_trans->req_q_addr = cpu_to_le64(dma_addr);
687	out_trans->req_q_size = cpu_to_le32(nelem);
688	out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());
689	out_trans->rsp_q_size = cpu_to_le32(nelem);
690	out_trans->options = cpu_to_le32(in_trans->options);
691
692	*user_len += in_trans->hdr.len;
693	msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));
694	msg->hdr.count = incr_le32(val: msg->hdr.count);
695
696	resources->buf = buf;
697	resources->dma_addr = dma_addr;
698	resources->total_size = size;
699	resources->nelem = nelem;
700	resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();
701	return 0;
702
703	free_dma:
704	dma_free_coherent(dev: &qdev->pdev->dev, size, cpu_addr: buf, dma_handle: dma_addr);
705	return ret;
706	}
707
708	static int encode_deactivate(struct qaic_device *qdev, void *trans,
709	u32 *user_len, struct qaic_user *usr)
710	{
711	struct qaic_manage_trans_deactivate *in_trans = trans;
712
713	if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)
714	return -EINVAL;
715
716	*user_len += in_trans->hdr.len;
717
718	return disable_dbc(qdev, dbc_id: in_trans->dbc_id, usr);
719	}
720
721	static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,
722	u32 *user_len)
723	{
724	struct qaic_manage_trans_status_to_dev *in_trans = trans;
725	struct wire_trans_status_to_dev *out_trans;
726	struct wrapper_msg *trans_wrapper;
727	struct wrapper_msg *wrapper;
728	struct wire_msg *msg;
729	u32 msg_hdr_len;
730
731	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
732	msg = &wrapper->msg;
733	msg_hdr_len = le32_to_cpu(msg->hdr.len);
734
735	if (size_add(addend1: msg_hdr_len, addend2: in_trans->hdr.len) > QAIC_MANAGE_WIRE_MSG_LENGTH)
736	return -ENOSPC;
737
738	trans_wrapper = add_wrapper(wrappers, size: sizeof(*trans_wrapper));
739	if (!trans_wrapper)
740	return -ENOMEM;
741
742	trans_wrapper->len = sizeof(*out_trans);
743	out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;
744
745	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);
746	out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);
747	msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);
748	msg->hdr.count = incr_le32(val: msg->hdr.count);
749	*user_len += in_trans->hdr.len;
750
751	return 0;
752	}
753
754	static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
755	struct wrapper_list *wrappers, struct ioctl_resources *resources,
756	struct qaic_user *usr)
757	{
758	struct qaic_manage_trans_hdr *trans_hdr;
759	struct wrapper_msg *wrapper;
760	struct wire_msg *msg;
761	u32 user_len = 0;
762	int ret;
763	int i;
764
765	if (!user_msg->count ||
766	user_msg->len < sizeof(*trans_hdr)) {
767	ret = -EINVAL;
768	goto out;
769	}
770
771	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
772	msg = &wrapper->msg;
773
774	msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));
775
776	if (resources->dma_chunk_id) {
777	ret = encode_dma(qdev, trans: resources->trans_hdr, wrappers, user_len: &user_len, resources, usr);
778	msg->hdr.count = cpu_to_le32(1);
779	goto out;
780	}
781
782	for (i = 0; i < user_msg->count; ++i) {
783	if (user_len > user_msg->len - sizeof(*trans_hdr)) {
784	ret = -EINVAL;
785	break;
786	}
787	trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);
788	if (trans_hdr->len < sizeof(trans_hdr) ||
789	size_add(addend1: user_len, addend2: trans_hdr->len) > user_msg->len) {
790	ret = -EINVAL;
791	break;
792	}
793
794	switch (trans_hdr->type) {
795	case QAIC_TRANS_PASSTHROUGH_FROM_USR:
796	ret = encode_passthrough(qdev, trans: trans_hdr, wrappers, user_len: &user_len);
797	break;
798	case QAIC_TRANS_DMA_XFER_FROM_USR:
799	ret = encode_dma(qdev, trans: trans_hdr, wrappers, user_len: &user_len, resources, usr);
800	break;
801	case QAIC_TRANS_ACTIVATE_FROM_USR:
802	ret = encode_activate(qdev, trans: trans_hdr, wrappers, user_len: &user_len, resources);
803	break;
804	case QAIC_TRANS_DEACTIVATE_FROM_USR:
805	ret = encode_deactivate(qdev, trans: trans_hdr, user_len: &user_len, usr);
806	break;
807	case QAIC_TRANS_STATUS_FROM_USR:
808	ret = encode_status(qdev, trans: trans_hdr, wrappers, user_len: &user_len);
809	break;
810	default:
811	ret = -EINVAL;
812	break;
813	}
814
815	if (ret)
816	goto out;
817	}
818
819	if (user_len != user_msg->len)
820	ret = -EINVAL;
821	out:
822	if (ret) {
823	free_dma_xfers(qdev, resources);
824	free_dbc_buf(qdev, resources);
825	return ret;
826	}
827
828	return 0;
829	}
830
831	static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
832	u32 *msg_len)
833	{
834	struct qaic_manage_trans_passthrough *out_trans;
835	struct wire_trans_passthrough *in_trans = trans;
836	u32 len;
837
838	out_trans = (void *)user_msg->data + user_msg->len;
839
840	len = le32_to_cpu(in_trans->hdr.len);
841	if (len % 8 != 0)
842	return -EINVAL;
843
844	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
845	return -ENOSPC;
846
847	memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));
848	user_msg->len += len;
849	*msg_len += len;
850	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
851	out_trans->hdr.len = len;
852
853	return 0;
854	}
855
856	static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
857	u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)
858	{
859	struct qaic_manage_trans_activate_from_dev *out_trans;
860	struct wire_trans_activate_from_dev *in_trans = trans;
861	u32 len;
862
863	out_trans = (void *)user_msg->data + user_msg->len;
864
865	len = le32_to_cpu(in_trans->hdr.len);
866	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
867	return -ENOSPC;
868
869	user_msg->len += len;
870	*msg_len += len;
871	out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);
872	out_trans->hdr.len = len;
873	out_trans->status = le32_to_cpu(in_trans->status);
874	out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);
875	out_trans->options = le64_to_cpu(in_trans->options);
876
877	if (!resources->buf)
878	/* how did we get an activate response without a request? */
879	return -EINVAL;
880
881	if (out_trans->dbc_id >= qdev->num_dbc)
882	/*
883	* The device assigned an invalid resource, which should never
884	* happen. Return an error so the user can try to recover.
885	*/
886	return -ENODEV;
887
888	if (out_trans->status)
889	/*
890	* Allocating resources failed on device side. This is not an
891	* expected behaviour, user is expected to handle this situation.
892	*/
893	return -ECANCELED;
894
895	resources->status = out_trans->status;
896	resources->dbc_id = out_trans->dbc_id;
897	save_dbc_buf(qdev, resources, usr);
898
899	return 0;
900	}
901
902	static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,
903	struct qaic_user *usr)
904	{
905	struct wire_trans_deactivate_from_dev *in_trans = trans;
906	u32 dbc_id = le32_to_cpu(in_trans->dbc_id);
907	u32 status = le32_to_cpu(in_trans->status);
908
909	if (dbc_id >= qdev->num_dbc)
910	/*
911	* The device assigned an invalid resource, which should never
912	* happen. Inject an error so the user can try to recover.
913	*/
914	return -ENODEV;
915
916	if (status) {
917	/*
918	* Releasing resources failed on the device side, which puts
919	* us in a bind since they may still be in use, so enable the
920	* dbc. User is expected to retry deactivation.
921	*/
922	enable_dbc(qdev, dbc_id, usr);
923	return -ECANCELED;
924	}
925
926	release_dbc(qdev, dbc_id);
927	set_dbc_state(qdev, dbc_id, state: DBC_STATE_IDLE);
928	*msg_len += sizeof(*in_trans);
929
930	return 0;
931	}
932
933	static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,
934	u32 *user_len, struct wire_msg *msg)
935	{
936	struct qaic_manage_trans_status_from_dev *out_trans;
937	struct wire_trans_status_from_dev *in_trans = trans;
938	u32 len;
939
940	out_trans = (void *)user_msg->data + user_msg->len;
941
942	len = le32_to_cpu(in_trans->hdr.len);
943	if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)
944	return -ENOSPC;
945
946	out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;
947	out_trans->hdr.len = len;
948	out_trans->major = le16_to_cpu(in_trans->major);
949	out_trans->minor = le16_to_cpu(in_trans->minor);
950	out_trans->status_flags = le64_to_cpu(in_trans->status_flags);
951	out_trans->status = le32_to_cpu(in_trans->status);
952	*user_len += le32_to_cpu(in_trans->hdr.len);
953	user_msg->len += len;
954
955	if (out_trans->status)
956	return -ECANCELED;
957	if (out_trans->status_flags & BIT(0) && !valid_crc(msg))
958	return -EPIPE;
959
960	return 0;
961	}
962
963	static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,
964	struct wire_msg *msg, struct ioctl_resources *resources,
965	struct qaic_user *usr)
966	{
967	u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);
968	struct wire_trans_hdr *trans_hdr;
969	u32 msg_len = 0;
970	int ret;
971	int i;
972
973	if (msg_hdr_len < sizeof(*trans_hdr) ||
974	msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)
975	return -EINVAL;
976
977	user_msg->len = 0;
978	user_msg->count = le32_to_cpu(msg->hdr.count);
979
980	for (i = 0; i < user_msg->count; ++i) {
981	u32 hdr_len;
982
983	if (msg_len > msg_hdr_len - sizeof(*trans_hdr))
984	return -EINVAL;
985
986	trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);
987	hdr_len = le32_to_cpu(trans_hdr->len);
988	if (hdr_len < sizeof(*trans_hdr) ||
989	size_add(addend1: msg_len, addend2: hdr_len) > msg_hdr_len)
990	return -EINVAL;
991
992	switch (le32_to_cpu(trans_hdr->type)) {
993	case QAIC_TRANS_PASSTHROUGH_FROM_DEV:
994	ret = decode_passthrough(qdev, trans: trans_hdr, user_msg, msg_len: &msg_len);
995	break;
996	case QAIC_TRANS_ACTIVATE_FROM_DEV:
997	ret = decode_activate(qdev, trans: trans_hdr, user_msg, msg_len: &msg_len, resources, usr);
998	break;
999	case QAIC_TRANS_DEACTIVATE_FROM_DEV:
1000	ret = decode_deactivate(qdev, trans: trans_hdr, msg_len: &msg_len, usr);
1001	break;
1002	case QAIC_TRANS_STATUS_FROM_DEV:
1003	ret = decode_status(qdev, trans: trans_hdr, user_msg, user_len: &msg_len, msg);
1004	break;
1005	default:
1006	return -EINVAL;
1007	}
1008
1009	if (ret)
1010	return ret;
1011	}
1012
1013	if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))
1014	return -EINVAL;
1015
1016	return 0;
1017	}
1018
1019	static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,
1020	bool ignore_signal)
1021	{
1022	struct xfer_queue_elem elem;
1023	struct wire_msg *out_buf;
1024	struct wrapper_msg *w;
1025	long ret = -EAGAIN;
1026	int xfer_count = 0;
1027	int retry_count;
1028
1029	/* Allow QAIC_BOOT state since we need to check control protocol version */
1030	if (qdev->dev_state == QAIC_OFFLINE) {
1031	mutex_unlock(lock: &qdev->cntl_mutex);
1032	return ERR_PTR(error: -ENODEV);
1033	}
1034
1035	/* Attempt to avoid a partial commit of a message */
1036	list_for_each_entry(w, &wrappers->list, list)
1037	xfer_count++;
1038
1039	for (retry_count = 0; retry_count < QAIC_MHI_RETRY_MAX; retry_count++) {
1040	if (xfer_count <= mhi_get_free_desc_count(mhi_dev: qdev->cntl_ch, dir: DMA_TO_DEVICE)) {
1041	ret = 0;
1042	break;
1043	}
1044	msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);
1045	if (signal_pending(current))
1046	break;
1047	}
1048
1049	if (ret) {
1050	mutex_unlock(lock: &qdev->cntl_mutex);
1051	return ERR_PTR(error: ret);
1052	}
1053
1054	elem.seq_num = seq_num;
1055	elem.buf = NULL;
1056	init_completion(x: &elem.xfer_done);
1057	if (likely(!qdev->cntl_lost_buf)) {
1058	/*
1059	* The max size of request to device is QAIC_MANAGE_WIRE_MSG_LENGTH.
1060	* The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.
1061	*/
1062	out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);
1063	if (!out_buf) {
1064	mutex_unlock(lock: &qdev->cntl_mutex);
1065	return ERR_PTR(error: -ENOMEM);
1066	}
1067
1068	ret = mhi_queue_buf(mhi_dev: qdev->cntl_ch, dir: DMA_FROM_DEVICE, buf: out_buf,
1069	QAIC_MANAGE_MAX_MSG_LENGTH, mflags: MHI_EOT);
1070	if (ret) {
1071	mutex_unlock(lock: &qdev->cntl_mutex);
1072	return ERR_PTR(error: ret);
1073	}
1074	} else {
1075	/*
1076	* we lost a buffer because we queued a recv buf, but then
1077	* queuing the corresponding tx buf failed. To try to avoid
1078	* a memory leak, lets reclaim it and use it for this
1079	* transaction.
1080	*/
1081	qdev->cntl_lost_buf = false;
1082	}
1083
1084	list_for_each_entry(w, &wrappers->list, list) {
1085	kref_get(kref: &w->ref_count);
1086	ret = mhi_queue_buf(mhi_dev: qdev->cntl_ch, dir: DMA_TO_DEVICE, buf: &w->msg, len: w->len,
1087	mflags: list_is_last(list: &w->list, head: &wrappers->list) ? MHI_EOT : MHI_CHAIN);
1088	if (ret) {
1089	qdev->cntl_lost_buf = true;
1090	kref_put(kref: &w->ref_count, release: free_wrapper);
1091	mutex_unlock(lock: &qdev->cntl_mutex);
1092	return ERR_PTR(error: ret);
1093	}
1094	}
1095
1096	list_add_tail(new: &elem.list, head: &qdev->cntl_xfer_list);
1097	mutex_unlock(lock: &qdev->cntl_mutex);
1098
1099	if (ignore_signal)
1100	ret = wait_for_completion_timeout(x: &elem.xfer_done, timeout: control_resp_timeout_s * HZ);
1101	else
1102	ret = wait_for_completion_interruptible_timeout(x: &elem.xfer_done,
1103	timeout: control_resp_timeout_s * HZ);
1104	/*
1105	* not using _interruptable because we have to cleanup or we'll
1106	* likely cause memory corruption
1107	*/
1108	mutex_lock(&qdev->cntl_mutex);
1109	if (!list_empty(head: &elem.list))
1110	list_del(entry: &elem.list);
1111	if (!ret && !elem.buf)
1112	ret = -ETIMEDOUT;
1113	else if (ret > 0 && !elem.buf)
1114	ret = -EIO;
1115	mutex_unlock(lock: &qdev->cntl_mutex);
1116
1117	if (ret < 0) {
1118	kfree(objp: elem.buf);
1119	return ERR_PTR(error: ret);
1120	} else if (!qdev->valid_crc(elem.buf)) {
1121	kfree(objp: elem.buf);
1122	return ERR_PTR(error: -EPIPE);
1123	}
1124
1125	return elem.buf;
1126	}
1127
1128	/* Add a transaction to abort the outstanding DMA continuation */
1129	static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)
1130	{
1131	struct wire_trans_dma_xfer *out_trans;
1132	u32 size = sizeof(*out_trans);
1133	struct wrapper_msg *wrapper;
1134	struct wrapper_msg *w;
1135	struct wire_msg *msg;
1136
1137	wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);
1138	msg = &wrapper->msg;
1139
1140	/* Remove all but the first wrapper which has the msg header */
1141	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1142	if (!list_is_first(list: &wrapper->list, head: &wrappers->list))
1143	kref_put(kref: &wrapper->ref_count, release: free_wrapper);
1144
1145	wrapper = add_wrapper(wrappers, size: sizeof(*wrapper));
1146
1147	if (!wrapper)
1148	return -ENOMEM;
1149
1150	out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;
1151	out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);
1152	out_trans->hdr.len = cpu_to_le32(size);
1153	out_trans->tag = cpu_to_le32(0);
1154	out_trans->count = cpu_to_le32(0);
1155	out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);
1156
1157	msg->hdr.len = cpu_to_le32(size + sizeof(*msg));
1158	msg->hdr.count = cpu_to_le32(1);
1159	wrapper->len = size;
1160
1161	return 0;
1162	}
1163
1164	static struct wrapper_list *alloc_wrapper_list(void)
1165	{
1166	struct wrapper_list *wrappers;
1167
1168	wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);
1169	if (!wrappers)
1170	return NULL;
1171	INIT_LIST_HEAD(list: &wrappers->list);
1172	spin_lock_init(&wrappers->lock);
1173
1174	return wrappers;
1175	}
1176
1177	static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,
1178	struct manage_msg *user_msg, struct ioctl_resources *resources,
1179	struct wire_msg **rsp)
1180	{
1181	struct wrapper_list *wrappers;
1182	struct wrapper_msg *wrapper;
1183	struct wrapper_msg *w;
1184	bool all_done = false;
1185	struct wire_msg *msg;
1186	int ret;
1187
1188	wrappers = alloc_wrapper_list();
1189	if (!wrappers)
1190	return -ENOMEM;
1191
1192	wrapper = add_wrapper(wrappers, size: sizeof(*wrapper));
1193	if (!wrapper) {
1194	kfree(objp: wrappers);
1195	return -ENOMEM;
1196	}
1197
1198	msg = &wrapper->msg;
1199	wrapper->len = sizeof(*msg);
1200
1201	ret = encode_message(qdev, user_msg, wrappers, resources, usr);
1202	if (ret && resources->dma_chunk_id)
1203	ret = abort_dma_cont(qdev, wrappers, dma_chunk_id: resources->dma_chunk_id);
1204	if (ret)
1205	goto encode_failed;
1206
1207	ret = mutex_lock_interruptible(&qdev->cntl_mutex);
1208	if (ret)
1209	goto lock_failed;
1210
1211	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1212	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1213
1214	if (usr) {
1215	msg->hdr.handle = cpu_to_le32(usr->handle);
1216	msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);
1217	} else {
1218	msg->hdr.handle = 0;
1219	msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);
1220	}
1221
1222	msg->hdr.padding = cpu_to_le32(0);
1223	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1224
1225	/* msg_xfer releases the mutex */
1226	*rsp = msg_xfer(qdev, wrappers, seq_num: qdev->next_seq_num - 1, ignore_signal: false);
1227	if (IS_ERR(ptr: *rsp))
1228	ret = PTR_ERR(ptr: *rsp);
1229
1230	lock_failed:
1231	free_dma_xfers(qdev, resources);
1232	encode_failed:
1233	spin_lock(lock: &wrappers->lock);
1234	list_for_each_entry_safe(wrapper, w, &wrappers->list, list)
1235	kref_put(kref: &wrapper->ref_count, release: free_wrapper);
1236	all_done = list_empty(head: &wrappers->list);
1237	spin_unlock(lock: &wrappers->lock);
1238	if (all_done)
1239	kfree(objp: wrappers);
1240
1241	return ret;
1242	}
1243
1244	static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)
1245	{
1246	struct wire_trans_dma_xfer_cont *dma_cont = NULL;
1247	struct ioctl_resources resources;
1248	struct wire_msg *rsp = NULL;
1249	int ret;
1250
1251	memset(&resources, 0, sizeof(struct ioctl_resources));
1252
1253	INIT_LIST_HEAD(list: &resources.dma_xfers);
1254
1255	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||
1256	user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))
1257	return -EINVAL;
1258
1259	dma_xfer_continue:
1260	ret = qaic_manage_msg_xfer(qdev, usr, user_msg, resources: &resources, rsp: &rsp);
1261	if (ret)
1262	return ret;
1263	/* dma_cont should be the only transaction if present */
1264	if (le32_to_cpu(rsp->hdr.count) == 1) {
1265	dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;
1266	if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)
1267	dma_cont = NULL;
1268	}
1269	if (dma_cont) {
1270	if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&
1271	le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {
1272	kfree(objp: rsp);
1273	goto dma_xfer_continue;
1274	}
1275
1276	ret = -EINVAL;
1277	goto dma_cont_failed;
1278	}
1279
1280	ret = decode_message(qdev, user_msg, msg: rsp, resources: &resources, usr);
1281
1282	dma_cont_failed:
1283	free_dbc_buf(qdev, resources: &resources);
1284	kfree(objp: rsp);
1285	return ret;
1286	}
1287
1288	int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
1289	{
1290	struct qaic_manage_msg *user_msg = data;
1291	struct qaic_device *qdev;
1292	struct manage_msg *msg;
1293	struct qaic_user *usr;
1294	u8 __user *user_data;
1295	int qdev_rcu_id;
1296	int usr_rcu_id;
1297	int ret;
1298
1299	if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH)
1300	return -EINVAL;
1301
1302	usr = file_priv->driver_priv;
1303
1304	usr_rcu_id = srcu_read_lock(ssp: &usr->qddev_lock);
1305	if (!usr->qddev) {
1306	srcu_read_unlock(ssp: &usr->qddev_lock, idx: usr_rcu_id);
1307	return -ENODEV;
1308	}
1309
1310	qdev = usr->qddev->qdev;
1311
1312	qdev_rcu_id = srcu_read_lock(ssp: &qdev->dev_lock);
1313	if (qdev->dev_state != QAIC_ONLINE) {
1314	srcu_read_unlock(ssp: &qdev->dev_lock, idx: qdev_rcu_id);
1315	srcu_read_unlock(ssp: &usr->qddev_lock, idx: usr_rcu_id);
1316	return -ENODEV;
1317	}
1318
1319	msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);
1320	if (!msg) {
1321	ret = -ENOMEM;
1322	goto out;
1323	}
1324
1325	msg->len = user_msg->len;
1326	msg->count = user_msg->count;
1327
1328	user_data = u64_to_user_ptr(user_msg->data);
1329
1330	if (copy_from_user(to: msg->data, from: user_data, n: user_msg->len)) {
1331	ret = -EFAULT;
1332	goto free_msg;
1333	}
1334
1335	ret = qaic_manage(qdev, usr, user_msg: msg);
1336
1337	/*
1338	* If the qaic_manage() is successful then we copy the message onto
1339	* userspace memory but we have an exception for -ECANCELED.
1340	* For -ECANCELED, it means that device has NACKed the message with a
1341	* status error code which userspace would like to know.
1342	*/
1343	if (ret == -ECANCELED || !ret) {
1344	if (copy_to_user(to: user_data, from: msg->data, n: msg->len)) {
1345	ret = -EFAULT;
1346	} else {
1347	user_msg->len = msg->len;
1348	user_msg->count = msg->count;
1349	}
1350	}
1351
1352	free_msg:
1353	kfree(objp: msg);
1354	out:
1355	srcu_read_unlock(ssp: &qdev->dev_lock, idx: qdev_rcu_id);
1356	srcu_read_unlock(ssp: &usr->qddev_lock, idx: usr_rcu_id);
1357	return ret;
1358	}
1359
1360	int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)
1361	{
1362	struct qaic_manage_trans_status_from_dev *status_result;
1363	struct qaic_manage_trans_status_to_dev *status_query;
1364	struct manage_msg *user_msg;
1365	int ret;
1366
1367	user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);
1368	if (!user_msg) {
1369	ret = -ENOMEM;
1370	goto out;
1371	}
1372	user_msg->len = sizeof(*status_query);
1373	user_msg->count = 1;
1374
1375	status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;
1376	status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;
1377	status_query->hdr.len = sizeof(status_query->hdr);
1378
1379	ret = qaic_manage(qdev, usr, user_msg);
1380	if (ret)
1381	goto kfree_user_msg;
1382	status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;
1383	*major = status_result->major;
1384	*minor = status_result->minor;
1385
1386	if (status_result->status_flags & BIT(0)) { /* device is using CRC */
1387	/* By default qdev->gen_crc is programmed to generate CRC */
1388	qdev->valid_crc = valid_crc;
1389	} else {
1390	/* By default qdev->valid_crc is programmed to bypass CRC */
1391	qdev->gen_crc = gen_crc_stub;
1392	}
1393
1394	kfree_user_msg:
1395	kfree(objp: user_msg);
1396	out:
1397	return ret;
1398	}
1399
1400	static void resp_worker(struct work_struct *work)
1401	{
1402	struct resp_work *resp = container_of(work, struct resp_work, work);
1403	struct qaic_device *qdev = resp->qdev;
1404	struct wire_msg *msg = resp->buf;
1405	struct xfer_queue_elem *elem;
1406	struct xfer_queue_elem *i;
1407	bool found = false;
1408
1409	mutex_lock(&qdev->cntl_mutex);
1410	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1411	if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {
1412	found = true;
1413	list_del_init(entry: &elem->list);
1414	elem->buf = msg;
1415	complete_all(&elem->xfer_done);
1416	break;
1417	}
1418	}
1419	mutex_unlock(lock: &qdev->cntl_mutex);
1420
1421	if (!found)
1422	/* request must have timed out, drop packet */
1423	kfree(objp: msg);
1424
1425	kfree(objp: resp);
1426	}
1427
1428	static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)
1429	{
1430	bool all_done = false;
1431
1432	spin_lock(lock: &wrappers->lock);
1433	kref_put(kref: &wrapper->ref_count, release: free_wrapper);
1434	all_done = list_empty(head: &wrappers->list);
1435	spin_unlock(lock: &wrappers->lock);
1436
1437	if (all_done)
1438	kfree(objp: wrappers);
1439	}
1440
1441	void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1442	{
1443	struct wire_msg *msg = mhi_result->buf_addr;
1444	struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);
1445
1446	free_wrapper_from_list(wrappers: wrapper->head, wrapper);
1447	}
1448
1449	void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)
1450	{
1451	struct qaic_device *qdev = dev_get_drvdata(dev: &mhi_dev->dev);
1452	struct wire_msg *msg = mhi_result->buf_addr;
1453	struct resp_work *resp;
1454
1455	if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {
1456	kfree(objp: msg);
1457	return;
1458	}
1459
1460	resp = kmalloc(sizeof(*resp), GFP_ATOMIC);
1461	if (!resp) {
1462	kfree(objp: msg);
1463	return;
1464	}
1465
1466	INIT_WORK(&resp->work, resp_worker);
1467	resp->qdev = qdev;
1468	resp->buf = msg;
1469	queue_work(wq: qdev->cntl_wq, work: &resp->work);
1470	}
1471
1472	int qaic_control_open(struct qaic_device *qdev)
1473	{
1474	if (!qdev->cntl_ch)
1475	return -ENODEV;
1476
1477	qdev->cntl_lost_buf = false;
1478	/*
1479	* By default qaic should assume that device has CRC enabled.
1480	* Qaic comes to know if device has CRC enabled or disabled during the
1481	* device status transaction, which is the first transaction performed
1482	* on control channel.
1483	*
1484	* So CRC validation of first device status transaction response is
1485	* ignored (by calling valid_crc_stub) and is done later during decoding
1486	* if device has CRC enabled.
1487	* Now that qaic knows whether device has CRC enabled or not it acts
1488	* accordingly.
1489	*/
1490	qdev->gen_crc = gen_crc;
1491	qdev->valid_crc = valid_crc_stub;
1492
1493	return mhi_prepare_for_transfer(mhi_dev: qdev->cntl_ch);
1494	}
1495
1496	void qaic_control_close(struct qaic_device *qdev)
1497	{
1498	mhi_unprepare_from_transfer(mhi_dev: qdev->cntl_ch);
1499	}
1500
1501	void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)
1502	{
1503	struct wire_trans_terminate_to_dev *trans;
1504	struct wrapper_list *wrappers;
1505	struct wrapper_msg *wrapper;
1506	struct wire_msg *msg;
1507	struct wire_msg *rsp;
1508
1509	wrappers = alloc_wrapper_list();
1510	if (!wrappers)
1511	return;
1512
1513	wrapper = add_wrapper(wrappers, size: sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));
1514	if (!wrapper)
1515	return;
1516
1517	msg = &wrapper->msg;
1518
1519	trans = (struct wire_trans_terminate_to_dev *)msg->data;
1520
1521	trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);
1522	trans->hdr.len = cpu_to_le32(sizeof(*trans));
1523	trans->handle = cpu_to_le32(usr->handle);
1524
1525	mutex_lock(&qdev->cntl_mutex);
1526	wrapper->len = sizeof(msg->hdr) + sizeof(*trans);
1527	msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;
1528	msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);
1529	msg->hdr.len = cpu_to_le32(wrapper->len);
1530	msg->hdr.count = cpu_to_le32(1);
1531	msg->hdr.handle = cpu_to_le32(usr->handle);
1532	msg->hdr.padding = cpu_to_le32(0);
1533	msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));
1534
1535	/*
1536	* msg_xfer releases the mutex
1537	* We don't care about the return of msg_xfer since we will not do
1538	* anything different based on what happens.
1539	* We ignore pending signals since one will be set if the user is
1540	* killed, and we need give the device a chance to cleanup, otherwise
1541	* DMA may still be in progress when we return.
1542	*/
1543	rsp = msg_xfer(qdev, wrappers, seq_num: qdev->next_seq_num - 1, ignore_signal: true);
1544	if (!IS_ERR(ptr: rsp))
1545	kfree(objp: rsp);
1546	free_wrapper_from_list(wrappers, wrapper);
1547	}
1548
1549	void wake_all_cntl(struct qaic_device *qdev)
1550	{
1551	struct xfer_queue_elem *elem;
1552	struct xfer_queue_elem *i;
1553
1554	mutex_lock(&qdev->cntl_mutex);
1555	list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {
1556	list_del_init(entry: &elem->list);
1557	complete_all(&elem->xfer_done);
1558	}
1559	mutex_unlock(lock: &qdev->cntl_mutex);
1560	}
1561