1	// SPDX-License-Identifier: GPL-2.0
2	// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3	// Copyright (c) 2018, Linaro Limited
4
5	#include <linux/completion.h>
6	#include <linux/device.h>
7	#include <linux/dma-buf.h>
8	#include <linux/dma-mapping.h>
9	#include <linux/dma-resv.h>
10	#include <linux/idr.h>
11	#include <linux/list.h>
12	#include <linux/miscdevice.h>
13	#include <linux/module.h>
14	#include <linux/of_address.h>
15	#include <linux/of.h>
16	#include <linux/platform_device.h>
17	#include <linux/sort.h>
18	#include <linux/of_platform.h>
19	#include <linux/rpmsg.h>
20	#include <linux/scatterlist.h>
21	#include <linux/slab.h>
22	#include <linux/firmware/qcom/qcom_scm.h>
23	#include <uapi/misc/fastrpc.h>
24	#include <linux/of_reserved_mem.h>
25
26	#define ADSP_DOMAIN_ID (0)
27	#define MDSP_DOMAIN_ID (1)
28	#define SDSP_DOMAIN_ID (2)
29	#define CDSP_DOMAIN_ID (3)
30	#define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
31	#define FASTRPC_MAX_SESSIONS 14
32	#define FASTRPC_MAX_VMIDS 16
33	#define FASTRPC_ALIGN 128
34	#define FASTRPC_MAX_FDLIST 16
35	#define FASTRPC_MAX_CRCLIST 64
36	#define FASTRPC_PHYS(p) ((p) & 0xffffffff)
37	#define FASTRPC_CTX_MAX (256)
38	#define FASTRPC_INIT_HANDLE 1
39	#define FASTRPC_DSP_UTILITIES_HANDLE 2
40	#define FASTRPC_CTXID_MASK (0xFF0)
41	#define INIT_FILELEN_MAX (2 * 1024 * 1024)
42	#define INIT_FILE_NAMELEN_MAX (128)
43	#define FASTRPC_DEVICE_NAME "fastrpc"
44
45	/* Add memory to static PD pool, protection thru XPU */
46	#define ADSP_MMAP_HEAP_ADDR 4
47	/* MAP static DMA buffer on DSP User PD */
48	#define ADSP_MMAP_DMA_BUFFER 6
49	/* Add memory to static PD pool protection thru hypervisor */
50	#define ADSP_MMAP_REMOTE_HEAP_ADDR 8
51	/* Add memory to userPD pool, for user heap */
52	#define ADSP_MMAP_ADD_PAGES 0x1000
53	/* Add memory to userPD pool, for LLC heap */
54	#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
55
56	#define DSP_UNSUPPORTED_API (0x80000414)
57	/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
58	#define FASTRPC_MAX_DSP_ATTRIBUTES (256)
59	#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
60
61	/* Retrives number of input buffers from the scalars parameter */
62	#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
63
64	/* Retrives number of output buffers from the scalars parameter */
65	#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
66
67	/* Retrives number of input handles from the scalars parameter */
68	#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
69
70	/* Retrives number of output handles from the scalars parameter */
71	#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
72
73	#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
74	REMOTE_SCALARS_OUTBUFS(sc) + \
75	REMOTE_SCALARS_INHANDLES(sc)+ \
76	REMOTE_SCALARS_OUTHANDLES(sc))
77	#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
78	(((attr & 0x07) << 29) | \
79	((method & 0x1f) << 24) | \
80	((in & 0xff) << 16) | \
81	((out & 0xff) << 8) | \
82	((oin & 0x0f) << 4) | \
83	(oout & 0x0f))
84
85	#define FASTRPC_SCALARS(method, in, out) \
86	FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
87
88	#define FASTRPC_CREATE_PROCESS_NARGS 6
89	#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3
90	/* Remote Method id table */
91	#define FASTRPC_RMID_INIT_ATTACH 0
92	#define FASTRPC_RMID_INIT_RELEASE 1
93	#define FASTRPC_RMID_INIT_MMAP 4
94	#define FASTRPC_RMID_INIT_MUNMAP 5
95	#define FASTRPC_RMID_INIT_CREATE 6
96	#define FASTRPC_RMID_INIT_CREATE_ATTR 7
97	#define FASTRPC_RMID_INIT_CREATE_STATIC 8
98	#define FASTRPC_RMID_INIT_MEM_MAP 10
99	#define FASTRPC_RMID_INIT_MEM_UNMAP 11
100
101	/* Protection Domain(PD) ids */
102	#define ROOT_PD (0)
103	#define USER_PD (1)
104	#define SENSORS_PD (2)
105
106	#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
107
108	static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",
109	"sdsp", "cdsp"};
110	struct fastrpc_phy_page {
111	u64 addr; /* physical address */
112	u64 size; /* size of contiguous region */
113	};
114
115	struct fastrpc_invoke_buf {
116	u32 num; /* number of contiguous regions */
117	u32 pgidx; /* index to start of contiguous region */
118	};
119
120	struct fastrpc_remote_dmahandle {
121	s32 fd; /* dma handle fd */
122	u32 offset; /* dma handle offset */
123	u32 len; /* dma handle length */
124	};
125
126	struct fastrpc_remote_buf {
127	u64 pv; /* buffer pointer */
128	u64 len; /* length of buffer */
129	};
130
131	union fastrpc_remote_arg {
132	struct fastrpc_remote_buf buf;
133	struct fastrpc_remote_dmahandle dma;
134	};
135
136	struct fastrpc_mmap_rsp_msg {
137	u64 vaddr;
138	};
139
140	struct fastrpc_mmap_req_msg {
141	s32 pgid;
142	u32 flags;
143	u64 vaddr;
144	s32 num;
145	};
146
147	struct fastrpc_mem_map_req_msg {
148	s32 pgid;
149	s32 fd;
150	s32 offset;
151	u32 flags;
152	u64 vaddrin;
153	s32 num;
154	s32 data_len;
155	};
156
157	struct fastrpc_munmap_req_msg {
158	s32 pgid;
159	u64 vaddr;
160	u64 size;
161	};
162
163	struct fastrpc_mem_unmap_req_msg {
164	s32 pgid;
165	s32 fd;
166	u64 vaddrin;
167	u64 len;
168	};
169
170	struct fastrpc_msg {
171	int pid; /* process group id */
172	int tid; /* thread id */
173	u64 ctx; /* invoke caller context */
174	u32 handle; /* handle to invoke */
175	u32 sc; /* scalars structure describing the data */
176	u64 addr; /* physical address */
177	u64 size; /* size of contiguous region */
178	};
179
180	struct fastrpc_invoke_rsp {
181	u64 ctx; /* invoke caller context */
182	int retval; /* invoke return value */
183	};
184
185	struct fastrpc_buf_overlap {
186	u64 start;
187	u64 end;
188	int raix;
189	u64 mstart;
190	u64 mend;
191	u64 offset;
192	};
193
194	struct fastrpc_buf {
195	struct fastrpc_user *fl;
196	struct dma_buf *dmabuf;
197	struct device *dev;
198	void *virt;
199	u64 phys;
200	u64 size;
201	/* Lock for dma buf attachments */
202	struct mutex lock;
203	struct list_head attachments;
204	/* mmap support */
205	struct list_head node; /* list of user requested mmaps */
206	uintptr_t raddr;
207	};
208
209	struct fastrpc_dma_buf_attachment {
210	struct device *dev;
211	struct sg_table sgt;
212	struct list_head node;
213	};
214
215	struct fastrpc_map {
216	struct list_head node;
217	struct fastrpc_user *fl;
218	int fd;
219	struct dma_buf *buf;
220	struct sg_table *table;
221	struct dma_buf_attachment *attach;
222	u64 phys;
223	u64 size;
224	void *va;
225	u64 len;
226	u64 raddr;
227	u32 attr;
228	struct kref refcount;
229	};
230
231	struct fastrpc_invoke_ctx {
232	int nscalars;
233	int nbufs;
234	int retval;
235	int pid;
236	int tgid;
237	u32 sc;
238	u32 *crc;
239	u64 ctxid;
240	u64 msg_sz;
241	struct kref refcount;
242	struct list_head node; /* list of ctxs */
243	struct completion work;
244	struct work_struct put_work;
245	struct fastrpc_msg msg;
246	struct fastrpc_user *fl;
247	union fastrpc_remote_arg *rpra;
248	struct fastrpc_map **maps;
249	struct fastrpc_buf *buf;
250	struct fastrpc_invoke_args *args;
251	struct fastrpc_buf_overlap *olaps;
252	struct fastrpc_channel_ctx *cctx;
253	};
254
255	struct fastrpc_session_ctx {
256	struct device *dev;
257	int sid;
258	bool used;
259	bool valid;
260	};
261
262	struct fastrpc_channel_ctx {
263	int domain_id;
264	int sesscount;
265	int vmcount;
266	struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];
267	struct rpmsg_device *rpdev;
268	struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];
269	spinlock_t lock;
270	struct idr ctx_idr;
271	struct list_head users;
272	struct kref refcount;
273	/* Flag if dsp attributes are cached */
274	bool valid_attributes;
275	u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];
276	struct fastrpc_device *secure_fdevice;
277	struct fastrpc_device *fdevice;
278	struct fastrpc_buf *remote_heap;
279	struct list_head invoke_interrupted_mmaps;
280	bool secure;
281	bool unsigned_support;
282	u64 dma_mask;
283	};
284
285	struct fastrpc_device {
286	struct fastrpc_channel_ctx *cctx;
287	struct miscdevice miscdev;
288	bool secure;
289	};
290
291	struct fastrpc_user {
292	struct list_head user;
293	struct list_head maps;
294	struct list_head pending;
295	struct list_head mmaps;
296
297	struct fastrpc_channel_ctx *cctx;
298	struct fastrpc_session_ctx *sctx;
299	struct fastrpc_buf *init_mem;
300
301	int tgid;
302	int pd;
303	bool is_secure_dev;
304	/* Lock for lists */
305	spinlock_t lock;
306	/* lock for allocations */
307	struct mutex mutex;
308	};
309
310	static void fastrpc_free_map(struct kref *ref)
311	{
312	struct fastrpc_map *map;
313
314	map = container_of(ref, struct fastrpc_map, refcount);
315
316	if (map->table) {
317	if (map->attr & FASTRPC_ATTR_SECUREMAP) {
318	struct qcom_scm_vmperm perm;
319	int vmid = map->fl->cctx->vmperms[0].vmid;
320	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
321	int err = 0;
322
323	perm.vmid = QCOM_SCM_VMID_HLOS;
324	perm.perm = QCOM_SCM_PERM_RWX;
325	err = qcom_scm_assign_mem(mem_addr: map->phys, mem_sz: map->size,
326	src: &src_perms, newvm: &perm, dest_cnt: 1);
327	if (err) {
328	dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
329	map->phys, map->size, err);
330	return;
331	}
332	}
333	dma_buf_unmap_attachment_unlocked(attach: map->attach, sg_table: map->table,
334	direction: DMA_BIDIRECTIONAL);
335	dma_buf_detach(dmabuf: map->buf, attach: map->attach);
336	dma_buf_put(dmabuf: map->buf);
337	}
338
339	if (map->fl) {
340	spin_lock(lock: &map->fl->lock);
341	list_del(entry: &map->node);
342	spin_unlock(lock: &map->fl->lock);
343	map->fl = NULL;
344	}
345
346	kfree(objp: map);
347	}
348
349	static void fastrpc_map_put(struct fastrpc_map *map)
350	{
351	if (map)
352	kref_put(kref: &map->refcount, release: fastrpc_free_map);
353	}
354
355	static int fastrpc_map_get(struct fastrpc_map *map)
356	{
357	if (!map)
358	return -ENOENT;
359
360	return kref_get_unless_zero(kref: &map->refcount) ? 0 : -ENOENT;
361	}
362
363
364	static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,
365	struct fastrpc_map **ppmap, bool take_ref)
366	{
367	struct fastrpc_session_ctx *sess = fl->sctx;
368	struct fastrpc_map *map = NULL;
369	int ret = -ENOENT;
370
371	spin_lock(lock: &fl->lock);
372	list_for_each_entry(map, &fl->maps, node) {
373	if (map->fd != fd)
374	continue;
375
376	if (take_ref) {
377	ret = fastrpc_map_get(map);
378	if (ret) {
379	dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",
380	__func__, fd, ret);
381	break;
382	}
383	}
384
385	*ppmap = map;
386	ret = 0;
387	break;
388	}
389	spin_unlock(lock: &fl->lock);
390
391	return ret;
392	}
393
394	static void fastrpc_buf_free(struct fastrpc_buf *buf)
395	{
396	dma_free_coherent(dev: buf->dev, size: buf->size, cpu_addr: buf->virt,
397	FASTRPC_PHYS(buf->phys));
398	kfree(objp: buf);
399	}
400
401	static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
402	u64 size, struct fastrpc_buf **obuf)
403	{
404	struct fastrpc_buf *buf;
405
406	buf = kzalloc(size: sizeof(*buf), GFP_KERNEL);
407	if (!buf)
408	return -ENOMEM;
409
410	INIT_LIST_HEAD(list: &buf->attachments);
411	INIT_LIST_HEAD(list: &buf->node);
412	mutex_init(&buf->lock);
413
414	buf->fl = fl;
415	buf->virt = NULL;
416	buf->phys = 0;
417	buf->size = size;
418	buf->dev = dev;
419	buf->raddr = 0;
420
421	buf->virt = dma_alloc_coherent(dev, size: buf->size, dma_handle: (dma_addr_t *)&buf->phys,
422	GFP_KERNEL);
423	if (!buf->virt) {
424	mutex_destroy(lock: &buf->lock);
425	kfree(objp: buf);
426	return -ENOMEM;
427	}
428
429	*obuf = buf;
430
431	return 0;
432	}
433
434	static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,
435	u64 size, struct fastrpc_buf **obuf)
436	{
437	int ret;
438	struct fastrpc_buf *buf;
439
440	ret = __fastrpc_buf_alloc(fl, dev, size, obuf);
441	if (ret)
442	return ret;
443
444	buf = *obuf;
445
446	if (fl->sctx && fl->sctx->sid)
447	buf->phys += ((u64)fl->sctx->sid << 32);
448
449	return 0;
450	}
451
452	static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,
453	u64 size, struct fastrpc_buf **obuf)
454	{
455	struct device *rdev = &fl->cctx->rpdev->dev;
456
457	return __fastrpc_buf_alloc(fl, dev: rdev, size, obuf);
458	}
459
460	static void fastrpc_channel_ctx_free(struct kref *ref)
461	{
462	struct fastrpc_channel_ctx *cctx;
463
464	cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);
465
466	kfree(objp: cctx);
467	}
468
469	static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)
470	{
471	kref_get(kref: &cctx->refcount);
472	}
473
474	static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
475	{
476	kref_put(kref: &cctx->refcount, release: fastrpc_channel_ctx_free);
477	}
478
479	static void fastrpc_context_free(struct kref *ref)
480	{
481	struct fastrpc_invoke_ctx *ctx;
482	struct fastrpc_channel_ctx *cctx;
483	unsigned long flags;
484	int i;
485
486	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
487	cctx = ctx->cctx;
488
489	for (i = 0; i < ctx->nbufs; i++)
490	fastrpc_map_put(map: ctx->maps[i]);
491
492	if (ctx->buf)
493	fastrpc_buf_free(buf: ctx->buf);
494
495	spin_lock_irqsave(&cctx->lock, flags);
496	idr_remove(&cctx->ctx_idr, id: ctx->ctxid >> 4);
497	spin_unlock_irqrestore(lock: &cctx->lock, flags);
498
499	kfree(objp: ctx->maps);
500	kfree(objp: ctx->olaps);
501	kfree(objp: ctx);
502
503	fastrpc_channel_ctx_put(cctx);
504	}
505
506	static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)
507	{
508	kref_get(kref: &ctx->refcount);
509	}
510
511	static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)
512	{
513	kref_put(kref: &ctx->refcount, release: fastrpc_context_free);
514	}
515
516	static void fastrpc_context_put_wq(struct work_struct *work)
517	{
518	struct fastrpc_invoke_ctx *ctx =
519	container_of(work, struct fastrpc_invoke_ctx, put_work);
520
521	fastrpc_context_put(ctx);
522	}
523
524	#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
525	static int olaps_cmp(const void *a, const void *b)
526	{
527	struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;
528	struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;
529	/* sort with lowest starting buffer first */
530	int st = CMP(pa->start, pb->start);
531	/* sort with highest ending buffer first */
532	int ed = CMP(pb->end, pa->end);
533
534	return st == 0 ? ed : st;
535	}
536
537	static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)
538	{
539	u64 max_end = 0;
540	int i;
541
542	for (i = 0; i < ctx->nbufs; ++i) {
543	ctx->olaps[i].start = ctx->args[i].ptr;
544	ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;
545	ctx->olaps[i].raix = i;
546	}
547
548	sort(base: ctx->olaps, num: ctx->nbufs, size: sizeof(*ctx->olaps), cmp_func: olaps_cmp, NULL);
549
550	for (i = 0; i < ctx->nbufs; ++i) {
551	/* Falling inside previous range */
552	if (ctx->olaps[i].start < max_end) {
553	ctx->olaps[i].mstart = max_end;
554	ctx->olaps[i].mend = ctx->olaps[i].end;
555	ctx->olaps[i].offset = max_end - ctx->olaps[i].start;
556
557	if (ctx->olaps[i].end > max_end) {
558	max_end = ctx->olaps[i].end;
559	} else {
560	ctx->olaps[i].mend = 0;
561	ctx->olaps[i].mstart = 0;
562	}
563
564	} else {
565	ctx->olaps[i].mend = ctx->olaps[i].end;
566	ctx->olaps[i].mstart = ctx->olaps[i].start;
567	ctx->olaps[i].offset = 0;
568	max_end = ctx->olaps[i].end;
569	}
570	}
571	}
572
573	static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
574	struct fastrpc_user *user, u32 kernel, u32 sc,
575	struct fastrpc_invoke_args *args)
576	{
577	struct fastrpc_channel_ctx *cctx = user->cctx;
578	struct fastrpc_invoke_ctx *ctx = NULL;
579	unsigned long flags;
580	int ret;
581
582	ctx = kzalloc(size: sizeof(*ctx), GFP_KERNEL);
583	if (!ctx)
584	return ERR_PTR(error: -ENOMEM);
585
586	INIT_LIST_HEAD(list: &ctx->node);
587	ctx->fl = user;
588	ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);
589	ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +
590	REMOTE_SCALARS_OUTBUFS(sc);
591
592	if (ctx->nscalars) {
593	ctx->maps = kcalloc(n: ctx->nscalars,
594	size: sizeof(*ctx->maps), GFP_KERNEL);
595	if (!ctx->maps) {
596	kfree(objp: ctx);
597	return ERR_PTR(error: -ENOMEM);
598	}
599	ctx->olaps = kcalloc(n: ctx->nscalars,
600	size: sizeof(*ctx->olaps), GFP_KERNEL);
601	if (!ctx->olaps) {
602	kfree(objp: ctx->maps);
603	kfree(objp: ctx);
604	return ERR_PTR(error: -ENOMEM);
605	}
606	ctx->args = args;
607	fastrpc_get_buff_overlaps(ctx);
608	}
609
610	/* Released in fastrpc_context_put() */
611	fastrpc_channel_ctx_get(cctx);
612
613	ctx->sc = sc;
614	ctx->retval = -1;
615	ctx->pid = current->pid;
616	ctx->tgid = user->tgid;
617	ctx->cctx = cctx;
618	init_completion(x: &ctx->work);
619	INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);
620
621	spin_lock(lock: &user->lock);
622	list_add_tail(new: &ctx->node, head: &user->pending);
623	spin_unlock(lock: &user->lock);
624
625	spin_lock_irqsave(&cctx->lock, flags);
626	ret = idr_alloc_cyclic(&cctx->ctx_idr, ptr: ctx, start: 1,
627	FASTRPC_CTX_MAX, GFP_ATOMIC);
628	if (ret < 0) {
629	spin_unlock_irqrestore(lock: &cctx->lock, flags);
630	goto err_idr;
631	}
632	ctx->ctxid = ret << 4;
633	spin_unlock_irqrestore(lock: &cctx->lock, flags);
634
635	kref_init(kref: &ctx->refcount);
636
637	return ctx;
638	err_idr:
639	spin_lock(lock: &user->lock);
640	list_del(entry: &ctx->node);
641	spin_unlock(lock: &user->lock);
642	fastrpc_channel_ctx_put(cctx);
643	kfree(objp: ctx->maps);
644	kfree(objp: ctx->olaps);
645	kfree(objp: ctx);
646
647	return ERR_PTR(error: ret);
648	}
649
650	static struct sg_table *
651	fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,
652	enum dma_data_direction dir)
653	{
654	struct fastrpc_dma_buf_attachment *a = attachment->priv;
655	struct sg_table *table;
656	int ret;
657
658	table = &a->sgt;
659
660	ret = dma_map_sgtable(dev: attachment->dev, sgt: table, dir, attrs: 0);
661	if (ret)
662	table = ERR_PTR(error: ret);
663	return table;
664	}
665
666	static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,
667	struct sg_table *table,
668	enum dma_data_direction dir)
669	{
670	dma_unmap_sgtable(dev: attach->dev, sgt: table, dir, attrs: 0);
671	}
672
673	static void fastrpc_release(struct dma_buf *dmabuf)
674	{
675	struct fastrpc_buf *buffer = dmabuf->priv;
676
677	fastrpc_buf_free(buf: buffer);
678	}
679
680	static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,
681	struct dma_buf_attachment *attachment)
682	{
683	struct fastrpc_dma_buf_attachment *a;
684	struct fastrpc_buf *buffer = dmabuf->priv;
685	int ret;
686
687	a = kzalloc(size: sizeof(*a), GFP_KERNEL);
688	if (!a)
689	return -ENOMEM;
690
691	ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,
692	FASTRPC_PHYS(buffer->phys), buffer->size);
693	if (ret < 0) {
694	dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");
695	kfree(objp: a);
696	return -EINVAL;
697	}
698
699	a->dev = attachment->dev;
700	INIT_LIST_HEAD(list: &a->node);
701	attachment->priv = a;
702
703	mutex_lock(&buffer->lock);
704	list_add(new: &a->node, head: &buffer->attachments);
705	mutex_unlock(lock: &buffer->lock);
706
707	return 0;
708	}
709
710	static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,
711	struct dma_buf_attachment *attachment)
712	{
713	struct fastrpc_dma_buf_attachment *a = attachment->priv;
714	struct fastrpc_buf *buffer = dmabuf->priv;
715
716	mutex_lock(&buffer->lock);
717	list_del(entry: &a->node);
718	mutex_unlock(lock: &buffer->lock);
719	sg_free_table(&a->sgt);
720	kfree(objp: a);
721	}
722
723	static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)
724	{
725	struct fastrpc_buf *buf = dmabuf->priv;
726
727	iosys_map_set_vaddr(map, vaddr: buf->virt);
728
729	return 0;
730	}
731
732	static int fastrpc_mmap(struct dma_buf *dmabuf,
733	struct vm_area_struct *vma)
734	{
735	struct fastrpc_buf *buf = dmabuf->priv;
736	size_t size = vma->vm_end - vma->vm_start;
737
738	dma_resv_assert_held(dmabuf->resv);
739
740	return dma_mmap_coherent(buf->dev, vma, buf->virt,
741	FASTRPC_PHYS(buf->phys), size);
742	}
743
744	static const struct dma_buf_ops fastrpc_dma_buf_ops = {
745	.attach = fastrpc_dma_buf_attach,
746	.detach = fastrpc_dma_buf_detatch,
747	.map_dma_buf = fastrpc_map_dma_buf,
748	.unmap_dma_buf = fastrpc_unmap_dma_buf,
749	.mmap = fastrpc_mmap,
750	.vmap = fastrpc_vmap,
751	.release = fastrpc_release,
752	};
753
754	static int fastrpc_map_create(struct fastrpc_user *fl, int fd,
755	u64 len, u32 attr, struct fastrpc_map **ppmap)
756	{
757	struct fastrpc_session_ctx *sess = fl->sctx;
758	struct fastrpc_map *map = NULL;
759	struct sg_table *table;
760	int err = 0;
761
762	if (!fastrpc_map_lookup(fl, fd, ppmap, take_ref: true))
763	return 0;
764
765	map = kzalloc(size: sizeof(*map), GFP_KERNEL);
766	if (!map)
767	return -ENOMEM;
768
769	INIT_LIST_HEAD(list: &map->node);
770	kref_init(kref: &map->refcount);
771
772	map->fl = fl;
773	map->fd = fd;
774	map->buf = dma_buf_get(fd);
775	if (IS_ERR(ptr: map->buf)) {
776	err = PTR_ERR(ptr: map->buf);
777	goto get_err;
778	}
779
780	map->attach = dma_buf_attach(dmabuf: map->buf, dev: sess->dev);
781	if (IS_ERR(ptr: map->attach)) {
782	dev_err(sess->dev, "Failed to attach dmabuf\n");
783	err = PTR_ERR(ptr: map->attach);
784	goto attach_err;
785	}
786
787	table = dma_buf_map_attachment_unlocked(attach: map->attach, direction: DMA_BIDIRECTIONAL);
788	if (IS_ERR(ptr: table)) {
789	err = PTR_ERR(ptr: table);
790	goto map_err;
791	}
792	map->table = table;
793
794	if (attr & FASTRPC_ATTR_SECUREMAP) {
795	map->phys = sg_phys(sg: map->table->sgl);
796	} else {
797	map->phys = sg_dma_address(map->table->sgl);
798	map->phys += ((u64)fl->sctx->sid << 32);
799	}
800	map->size = len;
801	map->va = sg_virt(sg: map->table->sgl);
802	map->len = len;
803
804	if (attr & FASTRPC_ATTR_SECUREMAP) {
805	/*
806	* If subsystem VMIDs are defined in DTSI, then do
807	* hyp_assign from HLOS to those VM(s)
808	*/
809	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
810	struct qcom_scm_vmperm dst_perms[2] = {0};
811
812	dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
813	dst_perms[0].perm = QCOM_SCM_PERM_RW;
814	dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;
815	dst_perms[1].perm = QCOM_SCM_PERM_RWX;
816	map->attr = attr;
817	err = qcom_scm_assign_mem(mem_addr: map->phys, mem_sz: (u64)map->size, src: &src_perms, newvm: dst_perms, dest_cnt: 2);
818	if (err) {
819	dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
820	map->phys, map->size, err);
821	goto map_err;
822	}
823	}
824	spin_lock(lock: &fl->lock);
825	list_add_tail(new: &map->node, head: &fl->maps);
826	spin_unlock(lock: &fl->lock);
827	*ppmap = map;
828
829	return 0;
830
831	map_err:
832	dma_buf_detach(dmabuf: map->buf, attach: map->attach);
833	attach_err:
834	dma_buf_put(dmabuf: map->buf);
835	get_err:
836	fastrpc_map_put(map);
837
838	return err;
839	}
840
841	/*
842	* Fastrpc payload buffer with metadata looks like:
843	*
844	* >>>>>> START of METADATA <<<<<<<<<
845	* +---------------------------------+
846	* | Arguments |
847	* | type:(union fastrpc_remote_arg)|
848	* | (0 - N) |
849	* +---------------------------------+
850	* | Invoke Buffer list |
851	* | type:(struct fastrpc_invoke_buf)|
852	* | (0 - N) |
853	* +---------------------------------+
854	* | Page info list |
855	* | type:(struct fastrpc_phy_page) |
856	* | (0 - N) |
857	* +---------------------------------+
858	* | Optional info |
859	* |(can be specific to SoC/Firmware)|
860	* +---------------------------------+
861	* >>>>>>>> END of METADATA <<<<<<<<<
862	* +---------------------------------+
863	* | Inline ARGS |
864	* | (0-N) |
865	* +---------------------------------+
866	*/
867
868	static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)
869	{
870	int size = 0;
871
872	size = (sizeof(struct fastrpc_remote_buf) +
873	sizeof(struct fastrpc_invoke_buf) +
874	sizeof(struct fastrpc_phy_page)) * ctx->nscalars +
875	sizeof(u64) * FASTRPC_MAX_FDLIST +
876	sizeof(u32) * FASTRPC_MAX_CRCLIST;
877
878	return size;
879	}
880
881	static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)
882	{
883	u64 size = 0;
884	int oix;
885
886	size = ALIGN(metalen, FASTRPC_ALIGN);
887	for (oix = 0; oix < ctx->nbufs; oix++) {
888	int i = ctx->olaps[oix].raix;
889
890	if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {
891
892	if (ctx->olaps[oix].offset == 0)
893	size = ALIGN(size, FASTRPC_ALIGN);
894
895	size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);
896	}
897	}
898
899	return size;
900	}
901
902	static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)
903	{
904	struct device *dev = ctx->fl->sctx->dev;
905	int i, err;
906
907	for (i = 0; i < ctx->nscalars; ++i) {
908
909	if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||
910	ctx->args[i].length == 0)
911	continue;
912
913	err = fastrpc_map_create(fl: ctx->fl, fd: ctx->args[i].fd,
914	len: ctx->args[i].length, attr: ctx->args[i].attr, ppmap: &ctx->maps[i]);
915	if (err) {
916	dev_err(dev, "Error Creating map %d\n", err);
917	return -EINVAL;
918	}
919
920	}
921	return 0;
922	}
923
924	static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)
925	{
926	return (struct fastrpc_invoke_buf *)(&pra[len]);
927	}
928
929	static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)
930	{
931	return (struct fastrpc_phy_page *)(&buf[len]);
932	}
933
934	static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
935	{
936	struct device *dev = ctx->fl->sctx->dev;
937	union fastrpc_remote_arg *rpra;
938	struct fastrpc_invoke_buf *list;
939	struct fastrpc_phy_page *pages;
940	int inbufs, i, oix, err = 0;
941	u64 len, rlen, pkt_size;
942	u64 pg_start, pg_end;
943	uintptr_t args;
944	int metalen;
945
946	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
947	metalen = fastrpc_get_meta_size(ctx);
948	pkt_size = fastrpc_get_payload_size(ctx, metalen);
949
950	err = fastrpc_create_maps(ctx);
951	if (err)
952	return err;
953
954	ctx->msg_sz = pkt_size;
955
956	err = fastrpc_buf_alloc(fl: ctx->fl, dev, size: pkt_size, obuf: &ctx->buf);
957	if (err)
958	return err;
959
960	memset(ctx->buf->virt, 0, pkt_size);
961	rpra = ctx->buf->virt;
962	list = fastrpc_invoke_buf_start(pra: rpra, len: ctx->nscalars);
963	pages = fastrpc_phy_page_start(buf: list, len: ctx->nscalars);
964	args = (uintptr_t)ctx->buf->virt + metalen;
965	rlen = pkt_size - metalen;
966	ctx->rpra = rpra;
967
968	for (oix = 0; oix < ctx->nbufs; ++oix) {
969	int mlen;
970
971	i = ctx->olaps[oix].raix;
972	len = ctx->args[i].length;
973
974	rpra[i].buf.pv = 0;
975	rpra[i].buf.len = len;
976	list[i].num = len ? 1 : 0;
977	list[i].pgidx = i;
978
979	if (!len)
980	continue;
981
982	if (ctx->maps[i]) {
983	struct vm_area_struct *vma = NULL;
984
985	rpra[i].buf.pv = (u64) ctx->args[i].ptr;
986	pages[i].addr = ctx->maps[i]->phys;
987
988	mmap_read_lock(current->mm);
989	vma = find_vma(current->mm, addr: ctx->args[i].ptr);
990	if (vma)
991	pages[i].addr += ctx->args[i].ptr -
992	vma->vm_start;
993	mmap_read_unlock(current->mm);
994
995	pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;
996	pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>
997	PAGE_SHIFT;
998	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
999
1000	} else {
1001
1002	if (ctx->olaps[oix].offset == 0) {
1003	rlen -= ALIGN(args, FASTRPC_ALIGN) - args;
1004	args = ALIGN(args, FASTRPC_ALIGN);
1005	}
1006
1007	mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;
1008
1009	if (rlen < mlen)
1010	goto bail;
1011
1012	rpra[i].buf.pv = args - ctx->olaps[oix].offset;
1013	pages[i].addr = ctx->buf->phys -
1014	ctx->olaps[oix].offset +
1015	(pkt_size - rlen);
1016	pages[i].addr = pages[i].addr & PAGE_MASK;
1017
1018	pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;
1019	pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
1020	pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;
1021	args = args + mlen;
1022	rlen -= mlen;
1023	}
1024
1025	if (i < inbufs && !ctx->maps[i]) {
1026	void *dst = (void *)(uintptr_t)rpra[i].buf.pv;
1027	void *src = (void *)(uintptr_t)ctx->args[i].ptr;
1028
1029	if (!kernel) {
1030	if (copy_from_user(to: dst, from: (void __user *)src,
1031	n: len)) {
1032	err = -EFAULT;
1033	goto bail;
1034	}
1035	} else {
1036	memcpy(dst, src, len);
1037	}
1038	}
1039	}
1040
1041	for (i = ctx->nbufs; i < ctx->nscalars; ++i) {
1042	list[i].num = ctx->args[i].length ? 1 : 0;
1043	list[i].pgidx = i;
1044	if (ctx->maps[i]) {
1045	pages[i].addr = ctx->maps[i]->phys;
1046	pages[i].size = ctx->maps[i]->size;
1047	}
1048	rpra[i].dma.fd = ctx->args[i].fd;
1049	rpra[i].dma.len = ctx->args[i].length;
1050	rpra[i].dma.offset = (u64) ctx->args[i].ptr;
1051	}
1052
1053	bail:
1054	if (err)
1055	dev_err(dev, "Error: get invoke args failed:%d\n", err);
1056
1057	return err;
1058	}
1059
1060	static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,
1061	u32 kernel)
1062	{
1063	union fastrpc_remote_arg *rpra = ctx->rpra;
1064	struct fastrpc_user *fl = ctx->fl;
1065	struct fastrpc_map *mmap = NULL;
1066	struct fastrpc_invoke_buf *list;
1067	struct fastrpc_phy_page *pages;
1068	u64 *fdlist;
1069	int i, inbufs, outbufs, handles;
1070
1071	inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
1072	outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
1073	handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);
1074	list = fastrpc_invoke_buf_start(pra: rpra, len: ctx->nscalars);
1075	pages = fastrpc_phy_page_start(buf: list, len: ctx->nscalars);
1076	fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
1077
1078	for (i = inbufs; i < ctx->nbufs; ++i) {
1079	if (!ctx->maps[i]) {
1080	void *src = (void *)(uintptr_t)rpra[i].buf.pv;
1081	void *dst = (void *)(uintptr_t)ctx->args[i].ptr;
1082	u64 len = rpra[i].buf.len;
1083
1084	if (!kernel) {
1085	if (copy_to_user(to: (void __user *)dst, from: src, n: len))
1086	return -EFAULT;
1087	} else {
1088	memcpy(dst, src, len);
1089	}
1090	}
1091	}
1092
1093	/* Clean up fdlist which is updated by DSP */
1094	for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {
1095	if (!fdlist[i])
1096	break;
1097	if (!fastrpc_map_lookup(fl, fd: (int)fdlist[i], ppmap: &mmap, take_ref: false))
1098	fastrpc_map_put(map: mmap);
1099	}
1100
1101	return 0;
1102	}
1103
1104	static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,
1105	struct fastrpc_invoke_ctx *ctx,
1106	u32 kernel, uint32_t handle)
1107	{
1108	struct fastrpc_channel_ctx *cctx;
1109	struct fastrpc_user *fl = ctx->fl;
1110	struct fastrpc_msg *msg = &ctx->msg;
1111	int ret;
1112
1113	cctx = fl->cctx;
1114	msg->pid = fl->tgid;
1115	msg->tid = current->pid;
1116
1117	if (kernel)
1118	msg->pid = 0;
1119
1120	msg->ctx = ctx->ctxid | fl->pd;
1121	msg->handle = handle;
1122	msg->sc = ctx->sc;
1123	msg->addr = ctx->buf ? ctx->buf->phys : 0;
1124	msg->size = roundup(ctx->msg_sz, PAGE_SIZE);
1125	fastrpc_context_get(ctx);
1126
1127	ret = rpmsg_send(ept: cctx->rpdev->ept, data: (void *)msg, len: sizeof(*msg));
1128
1129	if (ret)
1130	fastrpc_context_put(ctx);
1131
1132	return ret;
1133
1134	}
1135
1136	static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,
1137	u32 handle, u32 sc,
1138	struct fastrpc_invoke_args *args)
1139	{
1140	struct fastrpc_invoke_ctx *ctx = NULL;
1141	struct fastrpc_buf *buf, *b;
1142
1143	int err = 0;
1144
1145	if (!fl->sctx)
1146	return -EINVAL;
1147
1148	if (!fl->cctx->rpdev)
1149	return -EPIPE;
1150
1151	if (handle == FASTRPC_INIT_HANDLE && !kernel) {
1152	dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);
1153	return -EPERM;
1154	}
1155
1156	ctx = fastrpc_context_alloc(user: fl, kernel, sc, args);
1157	if (IS_ERR(ptr: ctx))
1158	return PTR_ERR(ptr: ctx);
1159
1160	err = fastrpc_get_args(kernel, ctx);
1161	if (err)
1162	goto bail;
1163
1164	/* make sure that all CPU memory writes are seen by DSP */
1165	dma_wmb();
1166	/* Send invoke buffer to remote dsp */
1167	err = fastrpc_invoke_send(sctx: fl->sctx, ctx, kernel, handle);
1168	if (err)
1169	goto bail;
1170
1171	if (kernel) {
1172	if (!wait_for_completion_timeout(x: &ctx->work, timeout: 10 * HZ))
1173	err = -ETIMEDOUT;
1174	} else {
1175	err = wait_for_completion_interruptible(x: &ctx->work);
1176	}
1177
1178	if (err)
1179	goto bail;
1180
1181	/* make sure that all memory writes by DSP are seen by CPU */
1182	dma_rmb();
1183	/* populate all the output buffers with results */
1184	err = fastrpc_put_args(ctx, kernel);
1185	if (err)
1186	goto bail;
1187
1188	/* Check the response from remote dsp */
1189	err = ctx->retval;
1190	if (err)
1191	goto bail;
1192
1193	bail:
1194	if (err != -ERESTARTSYS && err != -ETIMEDOUT) {
1195	/* We are done with this compute context */
1196	spin_lock(lock: &fl->lock);
1197	list_del(entry: &ctx->node);
1198	spin_unlock(lock: &fl->lock);
1199	fastrpc_context_put(ctx);
1200	}
1201
1202	if (err == -ERESTARTSYS) {
1203	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1204	list_del(entry: &buf->node);
1205	list_add_tail(new: &buf->node, head: &fl->cctx->invoke_interrupted_mmaps);
1206	}
1207	}
1208
1209	if (err)
1210	dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);
1211
1212	return err;
1213	}
1214
1215	static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)
1216	{
1217	/* Check if the device node is non-secure and channel is secure*/
1218	if (!fl->is_secure_dev && fl->cctx->secure) {
1219	/*
1220	* Allow untrusted applications to offload only to Unsigned PD when
1221	* channel is configured as secure and block untrusted apps on channel
1222	* that does not support unsigned PD offload
1223	*/
1224	if (!fl->cctx->unsigned_support || !unsigned_pd_request) {
1225	dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD");
1226	return true;
1227	}
1228	}
1229
1230	return false;
1231	}
1232
1233	static int fastrpc_init_create_static_process(struct fastrpc_user *fl,
1234	char __user *argp)
1235	{
1236	struct fastrpc_init_create_static init;
1237	struct fastrpc_invoke_args *args;
1238	struct fastrpc_phy_page pages[1];
1239	char *name;
1240	int err;
1241	struct {
1242	int pgid;
1243	u32 namelen;
1244	u32 pageslen;
1245	} inbuf;
1246	u32 sc;
1247
1248	args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, size: sizeof(*args), GFP_KERNEL);
1249	if (!args)
1250	return -ENOMEM;
1251
1252	if (copy_from_user(to: &init, from: argp, n: sizeof(init))) {
1253	err = -EFAULT;
1254	goto err;
1255	}
1256
1257	if (init.namelen > INIT_FILE_NAMELEN_MAX) {
1258	err = -EINVAL;
1259	goto err;
1260	}
1261
1262	name = kzalloc(size: init.namelen, GFP_KERNEL);
1263	if (!name) {
1264	err = -ENOMEM;
1265	goto err;
1266	}
1267
1268	if (copy_from_user(to: name, from: (void __user *)(uintptr_t)init.name, n: init.namelen)) {
1269	err = -EFAULT;
1270	goto err_name;
1271	}
1272
1273	if (!fl->cctx->remote_heap) {
1274	err = fastrpc_remote_heap_alloc(fl, dev: fl->sctx->dev, size: init.memlen,
1275	obuf: &fl->cctx->remote_heap);
1276	if (err)
1277	goto err_name;
1278
1279	/* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1280	if (fl->cctx->vmcount) {
1281	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1282
1283	err = qcom_scm_assign_mem(mem_addr: fl->cctx->remote_heap->phys,
1284	mem_sz: (u64)fl->cctx->remote_heap->size,
1285	src: &src_perms,
1286	newvm: fl->cctx->vmperms, dest_cnt: fl->cctx->vmcount);
1287	if (err) {
1288	dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d",
1289	fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1290	goto err_map;
1291	}
1292	}
1293	}
1294
1295	inbuf.pgid = fl->tgid;
1296	inbuf.namelen = init.namelen;
1297	inbuf.pageslen = 0;
1298	fl->pd = USER_PD;
1299
1300	args[0].ptr = (u64)(uintptr_t)&inbuf;
1301	args[0].length = sizeof(inbuf);
1302	args[0].fd = -1;
1303
1304	args[1].ptr = (u64)(uintptr_t)name;
1305	args[1].length = inbuf.namelen;
1306	args[1].fd = -1;
1307
1308	pages[0].addr = fl->cctx->remote_heap->phys;
1309	pages[0].size = fl->cctx->remote_heap->size;
1310
1311	args[2].ptr = (u64)(uintptr_t) pages;
1312	args[2].length = sizeof(*pages);
1313	args[2].fd = -1;
1314
1315	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);
1316
1317	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1318	sc, args);
1319	if (err)
1320	goto err_invoke;
1321
1322	kfree(objp: args);
1323
1324	return 0;
1325	err_invoke:
1326	if (fl->cctx->vmcount) {
1327	u64 src_perms = 0;
1328	struct qcom_scm_vmperm dst_perms;
1329	u32 i;
1330
1331	for (i = 0; i < fl->cctx->vmcount; i++)
1332	src_perms |= BIT(fl->cctx->vmperms[i].vmid);
1333
1334	dst_perms.vmid = QCOM_SCM_VMID_HLOS;
1335	dst_perms.perm = QCOM_SCM_PERM_RWX;
1336	err = qcom_scm_assign_mem(mem_addr: fl->cctx->remote_heap->phys,
1337	mem_sz: (u64)fl->cctx->remote_heap->size,
1338	src: &src_perms, newvm: &dst_perms, dest_cnt: 1);
1339	if (err)
1340	dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1341	fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);
1342	}
1343	err_map:
1344	fastrpc_buf_free(buf: fl->cctx->remote_heap);
1345	err_name:
1346	kfree(objp: name);
1347	err:
1348	kfree(objp: args);
1349
1350	return err;
1351	}
1352
1353	static int fastrpc_init_create_process(struct fastrpc_user *fl,
1354	char __user *argp)
1355	{
1356	struct fastrpc_init_create init;
1357	struct fastrpc_invoke_args *args;
1358	struct fastrpc_phy_page pages[1];
1359	struct fastrpc_map *map = NULL;
1360	struct fastrpc_buf *imem = NULL;
1361	int memlen;
1362	int err;
1363	struct {
1364	int pgid;
1365	u32 namelen;
1366	u32 filelen;
1367	u32 pageslen;
1368	u32 attrs;
1369	u32 siglen;
1370	} inbuf;
1371	u32 sc;
1372	bool unsigned_module = false;
1373
1374	args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, size: sizeof(*args), GFP_KERNEL);
1375	if (!args)
1376	return -ENOMEM;
1377
1378	if (copy_from_user(to: &init, from: argp, n: sizeof(init))) {
1379	err = -EFAULT;
1380	goto err;
1381	}
1382
1383	if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)
1384	unsigned_module = true;
1385
1386	if (is_session_rejected(fl, unsigned_pd_request: unsigned_module)) {
1387	err = -ECONNREFUSED;
1388	goto err;
1389	}
1390
1391	if (init.filelen > INIT_FILELEN_MAX) {
1392	err = -EINVAL;
1393	goto err;
1394	}
1395
1396	inbuf.pgid = fl->tgid;
1397	inbuf.namelen = strlen(current->comm) + 1;
1398	inbuf.filelen = init.filelen;
1399	inbuf.pageslen = 1;
1400	inbuf.attrs = init.attrs;
1401	inbuf.siglen = init.siglen;
1402	fl->pd = USER_PD;
1403
1404	if (init.filelen && init.filefd) {
1405	err = fastrpc_map_create(fl, fd: init.filefd, len: init.filelen, attr: 0, ppmap: &map);
1406	if (err)
1407	goto err;
1408	}
1409
1410	memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),
1411	1024 * 1024);
1412	err = fastrpc_buf_alloc(fl, dev: fl->sctx->dev, size: memlen,
1413	obuf: &imem);
1414	if (err)
1415	goto err_alloc;
1416
1417	fl->init_mem = imem;
1418	args[0].ptr = (u64)(uintptr_t)&inbuf;
1419	args[0].length = sizeof(inbuf);
1420	args[0].fd = -1;
1421
1422	args[1].ptr = (u64)(uintptr_t)current->comm;
1423	args[1].length = inbuf.namelen;
1424	args[1].fd = -1;
1425
1426	args[2].ptr = (u64) init.file;
1427	args[2].length = inbuf.filelen;
1428	args[2].fd = init.filefd;
1429
1430	pages[0].addr = imem->phys;
1431	pages[0].size = imem->size;
1432
1433	args[3].ptr = (u64)(uintptr_t) pages;
1434	args[3].length = 1 * sizeof(*pages);
1435	args[3].fd = -1;
1436
1437	args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;
1438	args[4].length = sizeof(inbuf.attrs);
1439	args[4].fd = -1;
1440
1441	args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;
1442	args[5].length = sizeof(inbuf.siglen);
1443	args[5].fd = -1;
1444
1445	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);
1446	if (init.attrs)
1447	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);
1448
1449	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1450	sc, args);
1451	if (err)
1452	goto err_invoke;
1453
1454	kfree(objp: args);
1455
1456	return 0;
1457
1458	err_invoke:
1459	fl->init_mem = NULL;
1460	fastrpc_buf_free(buf: imem);
1461	err_alloc:
1462	fastrpc_map_put(map);
1463	err:
1464	kfree(objp: args);
1465
1466	return err;
1467	}
1468
1469	static struct fastrpc_session_ctx *fastrpc_session_alloc(
1470	struct fastrpc_channel_ctx *cctx)
1471	{
1472	struct fastrpc_session_ctx *session = NULL;
1473	unsigned long flags;
1474	int i;
1475
1476	spin_lock_irqsave(&cctx->lock, flags);
1477	for (i = 0; i < cctx->sesscount; i++) {
1478	if (!cctx->session[i].used && cctx->session[i].valid) {
1479	cctx->session[i].used = true;
1480	session = &cctx->session[i];
1481	break;
1482	}
1483	}
1484	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1485
1486	return session;
1487	}
1488
1489	static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,
1490	struct fastrpc_session_ctx *session)
1491	{
1492	unsigned long flags;
1493
1494	spin_lock_irqsave(&cctx->lock, flags);
1495	session->used = false;
1496	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1497	}
1498
1499	static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)
1500	{
1501	struct fastrpc_invoke_args args[1];
1502	int tgid = 0;
1503	u32 sc;
1504
1505	tgid = fl->tgid;
1506	args[0].ptr = (u64)(uintptr_t) &tgid;
1507	args[0].length = sizeof(tgid);
1508	args[0].fd = -1;
1509	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);
1510
1511	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1512	sc, args: &args[0]);
1513	}
1514
1515	static int fastrpc_device_release(struct inode *inode, struct file *file)
1516	{
1517	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
1518	struct fastrpc_channel_ctx *cctx = fl->cctx;
1519	struct fastrpc_invoke_ctx *ctx, *n;
1520	struct fastrpc_map *map, *m;
1521	struct fastrpc_buf *buf, *b;
1522	unsigned long flags;
1523
1524	fastrpc_release_current_dsp_process(fl);
1525
1526	spin_lock_irqsave(&cctx->lock, flags);
1527	list_del(entry: &fl->user);
1528	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1529
1530	if (fl->init_mem)
1531	fastrpc_buf_free(buf: fl->init_mem);
1532
1533	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
1534	list_del(entry: &ctx->node);
1535	fastrpc_context_put(ctx);
1536	}
1537
1538	list_for_each_entry_safe(map, m, &fl->maps, node)
1539	fastrpc_map_put(map);
1540
1541	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
1542	list_del(entry: &buf->node);
1543	fastrpc_buf_free(buf);
1544	}
1545
1546	fastrpc_session_free(cctx, session: fl->sctx);
1547	fastrpc_channel_ctx_put(cctx);
1548
1549	mutex_destroy(lock: &fl->mutex);
1550	kfree(objp: fl);
1551	file->private_data = NULL;
1552
1553	return 0;
1554	}
1555
1556	static int fastrpc_device_open(struct inode *inode, struct file *filp)
1557	{
1558	struct fastrpc_channel_ctx *cctx;
1559	struct fastrpc_device *fdevice;
1560	struct fastrpc_user *fl = NULL;
1561	unsigned long flags;
1562
1563	fdevice = miscdev_to_fdevice(filp->private_data);
1564	cctx = fdevice->cctx;
1565
1566	fl = kzalloc(size: sizeof(*fl), GFP_KERNEL);
1567	if (!fl)
1568	return -ENOMEM;
1569
1570	/* Released in fastrpc_device_release() */
1571	fastrpc_channel_ctx_get(cctx);
1572
1573	filp->private_data = fl;
1574	spin_lock_init(&fl->lock);
1575	mutex_init(&fl->mutex);
1576	INIT_LIST_HEAD(list: &fl->pending);
1577	INIT_LIST_HEAD(list: &fl->maps);
1578	INIT_LIST_HEAD(list: &fl->mmaps);
1579	INIT_LIST_HEAD(list: &fl->user);
1580	fl->tgid = current->tgid;
1581	fl->cctx = cctx;
1582	fl->is_secure_dev = fdevice->secure;
1583
1584	fl->sctx = fastrpc_session_alloc(cctx);
1585	if (!fl->sctx) {
1586	dev_err(&cctx->rpdev->dev, "No session available\n");
1587	mutex_destroy(lock: &fl->mutex);
1588	kfree(objp: fl);
1589
1590	return -EBUSY;
1591	}
1592
1593	spin_lock_irqsave(&cctx->lock, flags);
1594	list_add_tail(new: &fl->user, head: &cctx->users);
1595	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1596
1597	return 0;
1598	}
1599
1600	static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)
1601	{
1602	struct fastrpc_alloc_dma_buf bp;
1603	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1604	struct fastrpc_buf *buf = NULL;
1605	int err;
1606
1607	if (copy_from_user(to: &bp, from: argp, n: sizeof(bp)))
1608	return -EFAULT;
1609
1610	err = fastrpc_buf_alloc(fl, dev: fl->sctx->dev, size: bp.size, obuf: &buf);
1611	if (err)
1612	return err;
1613	exp_info.ops = &fastrpc_dma_buf_ops;
1614	exp_info.size = bp.size;
1615	exp_info.flags = O_RDWR;
1616	exp_info.priv = buf;
1617	buf->dmabuf = dma_buf_export(exp_info: &exp_info);
1618	if (IS_ERR(ptr: buf->dmabuf)) {
1619	err = PTR_ERR(ptr: buf->dmabuf);
1620	fastrpc_buf_free(buf);
1621	return err;
1622	}
1623
1624	bp.fd = dma_buf_fd(dmabuf: buf->dmabuf, O_ACCMODE);
1625	if (bp.fd < 0) {
1626	dma_buf_put(dmabuf: buf->dmabuf);
1627	return -EINVAL;
1628	}
1629
1630	if (copy_to_user(to: argp, from: &bp, n: sizeof(bp))) {
1631	/*
1632	* The usercopy failed, but we can't do much about it, as
1633	* dma_buf_fd() already called fd_install() and made the
1634	* file descriptor accessible for the current process. It
1635	* might already be closed and dmabuf no longer valid when
1636	* we reach this point. Therefore "leak" the fd and rely on
1637	* the process exit path to do any required cleanup.
1638	*/
1639	return -EFAULT;
1640	}
1641
1642	return 0;
1643	}
1644
1645	static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)
1646	{
1647	struct fastrpc_invoke_args args[1];
1648	int tgid = fl->tgid;
1649	u32 sc;
1650
1651	args[0].ptr = (u64)(uintptr_t) &tgid;
1652	args[0].length = sizeof(tgid);
1653	args[0].fd = -1;
1654	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);
1655	fl->pd = pd;
1656
1657	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE,
1658	sc, args: &args[0]);
1659	}
1660
1661	static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)
1662	{
1663	struct fastrpc_invoke_args *args = NULL;
1664	struct fastrpc_invoke inv;
1665	u32 nscalars;
1666	int err;
1667
1668	if (copy_from_user(to: &inv, from: argp, n: sizeof(inv)))
1669	return -EFAULT;
1670
1671	/* nscalars is truncated here to max supported value */
1672	nscalars = REMOTE_SCALARS_LENGTH(inv.sc);
1673	if (nscalars) {
1674	args = kcalloc(n: nscalars, size: sizeof(*args), GFP_KERNEL);
1675	if (!args)
1676	return -ENOMEM;
1677
1678	if (copy_from_user(to: args, from: (void __user *)(uintptr_t)inv.args,
1679	n: nscalars * sizeof(*args))) {
1680	kfree(objp: args);
1681	return -EFAULT;
1682	}
1683	}
1684
1685	err = fastrpc_internal_invoke(fl, kernel: false, handle: inv.handle, sc: inv.sc, args);
1686	kfree(objp: args);
1687
1688	return err;
1689	}
1690
1691	static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,
1692	uint32_t dsp_attr_buf_len)
1693	{
1694	struct fastrpc_invoke_args args[2] = { 0 };
1695
1696	/* Capability filled in userspace */
1697	dsp_attr_buf[0] = 0;
1698
1699	args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;
1700	args[0].length = sizeof(dsp_attr_buf_len);
1701	args[0].fd = -1;
1702	args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];
1703	args[1].length = dsp_attr_buf_len;
1704	args[1].fd = -1;
1705	fl->pd = USER_PD;
1706
1707	return fastrpc_internal_invoke(fl, kernel: true, FASTRPC_DSP_UTILITIES_HANDLE,
1708	FASTRPC_SCALARS(0, 1, 1), args);
1709	}
1710
1711	static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,
1712	struct fastrpc_user *fl)
1713	{
1714	struct fastrpc_channel_ctx *cctx = fl->cctx;
1715	uint32_t attribute_id = cap->attribute_id;
1716	uint32_t *dsp_attributes;
1717	unsigned long flags;
1718	uint32_t domain = cap->domain;
1719	int err;
1720
1721	spin_lock_irqsave(&cctx->lock, flags);
1722	/* check if we already have queried dsp for attributes */
1723	if (cctx->valid_attributes) {
1724	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1725	goto done;
1726	}
1727	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1728
1729	dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);
1730	if (!dsp_attributes)
1731	return -ENOMEM;
1732
1733	err = fastrpc_get_info_from_dsp(fl, dsp_attr_buf: dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1734	if (err == DSP_UNSUPPORTED_API) {
1735	dev_info(&cctx->rpdev->dev,
1736	"Warning: DSP capabilities not supported on domain: %d\n", domain);
1737	kfree(objp: dsp_attributes);
1738	return -EOPNOTSUPP;
1739	} else if (err) {
1740	dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);
1741	kfree(objp: dsp_attributes);
1742	return err;
1743	}
1744
1745	spin_lock_irqsave(&cctx->lock, flags);
1746	memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);
1747	cctx->valid_attributes = true;
1748	spin_unlock_irqrestore(lock: &cctx->lock, flags);
1749	kfree(objp: dsp_attributes);
1750	done:
1751	cap->capability = cctx->dsp_attributes[attribute_id];
1752	return 0;
1753	}
1754
1755	static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)
1756	{
1757	struct fastrpc_ioctl_capability cap = {0};
1758	int err = 0;
1759
1760	if (copy_from_user(to: &cap, from: argp, n: sizeof(cap)))
1761	return -EFAULT;
1762
1763	cap.capability = 0;
1764	if (cap.domain >= FASTRPC_DEV_MAX) {
1765	dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",
1766	cap.domain, err);
1767	return -ECHRNG;
1768	}
1769
1770	/* Fastrpc Capablities does not support modem domain */
1771	if (cap.domain == MDSP_DOMAIN_ID) {
1772	dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);
1773	return -ECHRNG;
1774	}
1775
1776	if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {
1777	dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",
1778	cap.attribute_id, err);
1779	return -EOVERFLOW;
1780	}
1781
1782	err = fastrpc_get_info_from_kernel(cap: &cap, fl);
1783	if (err)
1784	return err;
1785
1786	if (copy_to_user(to: argp, from: &cap.capability, n: sizeof(cap.capability)))
1787	return -EFAULT;
1788
1789	return 0;
1790	}
1791
1792	static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)
1793	{
1794	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1795	struct fastrpc_munmap_req_msg req_msg;
1796	struct device *dev = fl->sctx->dev;
1797	int err;
1798	u32 sc;
1799
1800	req_msg.pgid = fl->tgid;
1801	req_msg.size = buf->size;
1802	req_msg.vaddr = buf->raddr;
1803
1804	args[0].ptr = (u64) (uintptr_t) &req_msg;
1805	args[0].length = sizeof(req_msg);
1806
1807	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);
1808	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1809	args: &args[0]);
1810	if (!err) {
1811	dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);
1812	spin_lock(lock: &fl->lock);
1813	list_del(entry: &buf->node);
1814	spin_unlock(lock: &fl->lock);
1815	fastrpc_buf_free(buf);
1816	} else {
1817	dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);
1818	}
1819
1820	return err;
1821	}
1822
1823	static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)
1824	{
1825	struct fastrpc_buf *buf = NULL, *iter, *b;
1826	struct fastrpc_req_munmap req;
1827	struct device *dev = fl->sctx->dev;
1828
1829	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1830	return -EFAULT;
1831
1832	spin_lock(lock: &fl->lock);
1833	list_for_each_entry_safe(iter, b, &fl->mmaps, node) {
1834	if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {
1835	buf = iter;
1836	break;
1837	}
1838	}
1839	spin_unlock(lock: &fl->lock);
1840
1841	if (!buf) {
1842	dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1843	req.vaddrout, req.size);
1844	return -EINVAL;
1845	}
1846
1847	return fastrpc_req_munmap_impl(fl, buf);
1848	}
1849
1850	static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)
1851	{
1852	struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };
1853	struct fastrpc_buf *buf = NULL;
1854	struct fastrpc_mmap_req_msg req_msg;
1855	struct fastrpc_mmap_rsp_msg rsp_msg;
1856	struct fastrpc_phy_page pages;
1857	struct fastrpc_req_mmap req;
1858	struct device *dev = fl->sctx->dev;
1859	int err;
1860	u32 sc;
1861
1862	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
1863	return -EFAULT;
1864
1865	if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {
1866	dev_err(dev, "flag not supported 0x%x\n", req.flags);
1867
1868	return -EINVAL;
1869	}
1870
1871	if (req.vaddrin) {
1872	dev_err(dev, "adding user allocated pages is not supported\n");
1873	return -EINVAL;
1874	}
1875
1876	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
1877	err = fastrpc_remote_heap_alloc(fl, dev, size: req.size, obuf: &buf);
1878	else
1879	err = fastrpc_buf_alloc(fl, dev, size: req.size, obuf: &buf);
1880
1881	if (err) {
1882	dev_err(dev, "failed to allocate buffer\n");
1883	return err;
1884	}
1885
1886	req_msg.pgid = fl->tgid;
1887	req_msg.flags = req.flags;
1888	req_msg.vaddr = req.vaddrin;
1889	req_msg.num = sizeof(pages);
1890
1891	args[0].ptr = (u64) (uintptr_t) &req_msg;
1892	args[0].length = sizeof(req_msg);
1893
1894	pages.addr = buf->phys;
1895	pages.size = buf->size;
1896
1897	args[1].ptr = (u64) (uintptr_t) &pages;
1898	args[1].length = sizeof(pages);
1899
1900	args[2].ptr = (u64) (uintptr_t) &rsp_msg;
1901	args[2].length = sizeof(rsp_msg);
1902
1903	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);
1904	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1905	args: &args[0]);
1906	if (err) {
1907	dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);
1908	goto err_invoke;
1909	}
1910
1911	/* update the buffer to be able to deallocate the memory on the DSP */
1912	buf->raddr = (uintptr_t) rsp_msg.vaddr;
1913
1914	/* let the client know the address to use */
1915	req.vaddrout = rsp_msg.vaddr;
1916
1917	/* Add memory to static PD pool, protection thru hypervisor */
1918	if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {
1919	u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
1920
1921	err = qcom_scm_assign_mem(mem_addr: buf->phys, mem_sz: (u64)buf->size,
1922	src: &src_perms, newvm: fl->cctx->vmperms, dest_cnt: fl->cctx->vmcount);
1923	if (err) {
1924	dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1925	buf->phys, buf->size, err);
1926	goto err_assign;
1927	}
1928	}
1929
1930	spin_lock(lock: &fl->lock);
1931	list_add_tail(new: &buf->node, head: &fl->mmaps);
1932	spin_unlock(lock: &fl->lock);
1933
1934	if (copy_to_user(to: (void __user *)argp, from: &req, n: sizeof(req))) {
1935	err = -EFAULT;
1936	goto err_assign;
1937	}
1938
1939	dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1940	buf->raddr, buf->size);
1941
1942	return 0;
1943
1944	err_assign:
1945	fastrpc_req_munmap_impl(fl, buf);
1946	err_invoke:
1947	fastrpc_buf_free(buf);
1948
1949	return err;
1950	}
1951
1952	static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)
1953	{
1954	struct fastrpc_invoke_args args[1] = { [0] = { 0 } };
1955	struct fastrpc_map *map = NULL, *iter, *m;
1956	struct fastrpc_mem_unmap_req_msg req_msg = { 0 };
1957	int err = 0;
1958	u32 sc;
1959	struct device *dev = fl->sctx->dev;
1960
1961	spin_lock(lock: &fl->lock);
1962	list_for_each_entry_safe(iter, m, &fl->maps, node) {
1963	if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {
1964	map = iter;
1965	break;
1966	}
1967	}
1968
1969	spin_unlock(lock: &fl->lock);
1970
1971	if (!map) {
1972	dev_err(dev, "map not in list\n");
1973	return -EINVAL;
1974	}
1975
1976	req_msg.pgid = fl->tgid;
1977	req_msg.len = map->len;
1978	req_msg.vaddrin = map->raddr;
1979	req_msg.fd = map->fd;
1980
1981	args[0].ptr = (u64) (uintptr_t) &req_msg;
1982	args[0].length = sizeof(req_msg);
1983
1984	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);
1985	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc,
1986	args: &args[0]);
1987	if (err) {
1988	dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr);
1989	return err;
1990	}
1991	fastrpc_map_put(map);
1992
1993	return 0;
1994	}
1995
1996	static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)
1997	{
1998	struct fastrpc_mem_unmap req;
1999
2000	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
2001	return -EFAULT;
2002
2003	return fastrpc_req_mem_unmap_impl(fl, req: &req);
2004	}
2005
2006	static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)
2007	{
2008	struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };
2009	struct fastrpc_mem_map_req_msg req_msg = { 0 };
2010	struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };
2011	struct fastrpc_mem_unmap req_unmap = { 0 };
2012	struct fastrpc_phy_page pages = { 0 };
2013	struct fastrpc_mem_map req;
2014	struct device *dev = fl->sctx->dev;
2015	struct fastrpc_map *map = NULL;
2016	int err;
2017	u32 sc;
2018
2019	if (copy_from_user(to: &req, from: argp, n: sizeof(req)))
2020	return -EFAULT;
2021
2022	/* create SMMU mapping */
2023	err = fastrpc_map_create(fl, fd: req.fd, len: req.length, attr: 0, ppmap: &map);
2024	if (err) {
2025	dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);
2026	return err;
2027	}
2028
2029	req_msg.pgid = fl->tgid;
2030	req_msg.fd = req.fd;
2031	req_msg.offset = req.offset;
2032	req_msg.vaddrin = req.vaddrin;
2033	map->va = (void *) (uintptr_t) req.vaddrin;
2034	req_msg.flags = req.flags;
2035	req_msg.num = sizeof(pages);
2036	req_msg.data_len = 0;
2037
2038	args[0].ptr = (u64) (uintptr_t) &req_msg;
2039	args[0].length = sizeof(req_msg);
2040
2041	pages.addr = map->phys;
2042	pages.size = map->size;
2043
2044	args[1].ptr = (u64) (uintptr_t) &pages;
2045	args[1].length = sizeof(pages);
2046
2047	args[2].ptr = (u64) (uintptr_t) &pages;
2048	args[2].length = 0;
2049
2050	args[3].ptr = (u64) (uintptr_t) &rsp_msg;
2051	args[3].length = sizeof(rsp_msg);
2052
2053	sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);
2054	err = fastrpc_internal_invoke(fl, kernel: true, FASTRPC_INIT_HANDLE, sc, args: &args[0]);
2055	if (err) {
2056	dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2057	req.fd, req.vaddrin, map->size);
2058	goto err_invoke;
2059	}
2060
2061	/* update the buffer to be able to deallocate the memory on the DSP */
2062	map->raddr = rsp_msg.vaddr;
2063
2064	/* let the client know the address to use */
2065	req.vaddrout = rsp_msg.vaddr;
2066
2067	if (copy_to_user(to: (void __user *)argp, from: &req, n: sizeof(req))) {
2068	/* unmap the memory and release the buffer */
2069	req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;
2070	req_unmap.length = map->size;
2071	fastrpc_req_mem_unmap_impl(fl, req: &req_unmap);
2072	return -EFAULT;
2073	}
2074
2075	return 0;
2076
2077	err_invoke:
2078	fastrpc_map_put(map);
2079
2080	return err;
2081	}
2082
2083	static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,
2084	unsigned long arg)
2085	{
2086	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
2087	char __user *argp = (char __user *)arg;
2088	int err;
2089
2090	switch (cmd) {
2091	case FASTRPC_IOCTL_INVOKE:
2092	err = fastrpc_invoke(fl, argp);
2093	break;
2094	case FASTRPC_IOCTL_INIT_ATTACH:
2095	err = fastrpc_init_attach(fl, ROOT_PD);
2096	break;
2097	case FASTRPC_IOCTL_INIT_ATTACH_SNS:
2098	err = fastrpc_init_attach(fl, SENSORS_PD);
2099	break;
2100	case FASTRPC_IOCTL_INIT_CREATE_STATIC:
2101	err = fastrpc_init_create_static_process(fl, argp);
2102	break;
2103	case FASTRPC_IOCTL_INIT_CREATE:
2104	err = fastrpc_init_create_process(fl, argp);
2105	break;
2106	case FASTRPC_IOCTL_ALLOC_DMA_BUFF:
2107	err = fastrpc_dmabuf_alloc(fl, argp);
2108	break;
2109	case FASTRPC_IOCTL_MMAP:
2110	err = fastrpc_req_mmap(fl, argp);
2111	break;
2112	case FASTRPC_IOCTL_MUNMAP:
2113	err = fastrpc_req_munmap(fl, argp);
2114	break;
2115	case FASTRPC_IOCTL_MEM_MAP:
2116	err = fastrpc_req_mem_map(fl, argp);
2117	break;
2118	case FASTRPC_IOCTL_MEM_UNMAP:
2119	err = fastrpc_req_mem_unmap(fl, argp);
2120	break;
2121	case FASTRPC_IOCTL_GET_DSP_INFO:
2122	err = fastrpc_get_dsp_info(fl, argp);
2123	break;
2124	default:
2125	err = -ENOTTY;
2126	break;
2127	}
2128
2129	return err;
2130	}
2131
2132	static const struct file_operations fastrpc_fops = {
2133	.open = fastrpc_device_open,
2134	.release = fastrpc_device_release,
2135	.unlocked_ioctl = fastrpc_device_ioctl,
2136	.compat_ioctl = fastrpc_device_ioctl,
2137	};
2138
2139	static int fastrpc_cb_probe(struct platform_device *pdev)
2140	{
2141	struct fastrpc_channel_ctx *cctx;
2142	struct fastrpc_session_ctx *sess;
2143	struct device *dev = &pdev->dev;
2144	int i, sessions = 0;
2145	unsigned long flags;
2146	int rc;
2147
2148	cctx = dev_get_drvdata(dev: dev->parent);
2149	if (!cctx)
2150	return -EINVAL;
2151
2152	of_property_read_u32(np: dev->of_node, propname: "qcom,nsessions", out_value: &sessions);
2153
2154	spin_lock_irqsave(&cctx->lock, flags);
2155	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
2156	dev_err(&pdev->dev, "too many sessions\n");
2157	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2158	return -ENOSPC;
2159	}
2160	sess = &cctx->session[cctx->sesscount++];
2161	sess->used = false;
2162	sess->valid = true;
2163	sess->dev = dev;
2164	dev_set_drvdata(dev, data: sess);
2165
2166	if (of_property_read_u32(np: dev->of_node, propname: "reg", out_value: &sess->sid))
2167	dev_info(dev, "FastRPC Session ID not specified in DT\n");
2168
2169	if (sessions > 0) {
2170	struct fastrpc_session_ctx *dup_sess;
2171
2172	for (i = 1; i < sessions; i++) {
2173	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)
2174	break;
2175	dup_sess = &cctx->session[cctx->sesscount++];
2176	memcpy(dup_sess, sess, sizeof(*dup_sess));
2177	}
2178	}
2179	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2180	rc = dma_set_mask(dev, DMA_BIT_MASK(32));
2181	if (rc) {
2182	dev_err(dev, "32-bit DMA enable failed\n");
2183	return rc;
2184	}
2185
2186	return 0;
2187	}
2188
2189	static void fastrpc_cb_remove(struct platform_device *pdev)
2190	{
2191	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: pdev->dev.parent);
2192	struct fastrpc_session_ctx *sess = dev_get_drvdata(dev: &pdev->dev);
2193	unsigned long flags;
2194	int i;
2195
2196	spin_lock_irqsave(&cctx->lock, flags);
2197	for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {
2198	if (cctx->session[i].sid == sess->sid) {
2199	cctx->session[i].valid = false;
2200	cctx->sesscount--;
2201	}
2202	}
2203	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2204	}
2205
2206	static const struct of_device_id fastrpc_match_table[] = {
2207	{ .compatible = "qcom,fastrpc-compute-cb", },
2208	{}
2209	};
2210
2211	static struct platform_driver fastrpc_cb_driver = {
2212	.probe = fastrpc_cb_probe,
2213	.remove_new = fastrpc_cb_remove,
2214	.driver = {
2215	.name = "qcom,fastrpc-cb",
2216	.of_match_table = fastrpc_match_table,
2217	.suppress_bind_attrs = true,
2218	},
2219	};
2220
2221	static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,
2222	bool is_secured, const char *domain)
2223	{
2224	struct fastrpc_device *fdev;
2225	int err;
2226
2227	fdev = devm_kzalloc(dev, size: sizeof(*fdev), GFP_KERNEL);
2228	if (!fdev)
2229	return -ENOMEM;
2230
2231	fdev->secure = is_secured;
2232	fdev->cctx = cctx;
2233	fdev->miscdev.minor = MISC_DYNAMIC_MINOR;
2234	fdev->miscdev.fops = &fastrpc_fops;
2235	fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, fmt: "fastrpc-%s%s",
2236	domain, is_secured ? "-secure" : "");
2237	if (!fdev->miscdev.name)
2238	return -ENOMEM;
2239
2240	err = misc_register(misc: &fdev->miscdev);
2241	if (!err) {
2242	if (is_secured)
2243	cctx->secure_fdevice = fdev;
2244	else
2245	cctx->fdevice = fdev;
2246	}
2247
2248	return err;
2249	}
2250
2251	static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
2252	{
2253	struct device *rdev = &rpdev->dev;
2254	struct fastrpc_channel_ctx *data;
2255	int i, err, domain_id = -1, vmcount;
2256	const char *domain;
2257	bool secure_dsp;
2258	unsigned int vmids[FASTRPC_MAX_VMIDS];
2259
2260	err = of_property_read_string(np: rdev->of_node, propname: "label", out_string: &domain);
2261	if (err) {
2262	dev_info(rdev, "FastRPC Domain not specified in DT\n");
2263	return err;
2264	}
2265
2266	for (i = 0; i <= CDSP_DOMAIN_ID; i++) {
2267	if (!strcmp(domains[i], domain)) {
2268	domain_id = i;
2269	break;
2270	}
2271	}
2272
2273	if (domain_id < 0) {
2274	dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);
2275	return -EINVAL;
2276	}
2277
2278	if (of_reserved_mem_device_init_by_idx(dev: rdev, np: rdev->of_node, idx: 0))
2279	dev_info(rdev, "no reserved DMA memory for FASTRPC\n");
2280
2281	vmcount = of_property_read_variable_u32_array(np: rdev->of_node,
2282	propname: "qcom,vmids", out_values: &vmids[0], sz_min: 0, FASTRPC_MAX_VMIDS);
2283	if (vmcount < 0)
2284	vmcount = 0;
2285	else if (!qcom_scm_is_available())
2286	return -EPROBE_DEFER;
2287
2288	data = kzalloc(size: sizeof(*data), GFP_KERNEL);
2289	if (!data)
2290	return -ENOMEM;
2291
2292	if (vmcount) {
2293	data->vmcount = vmcount;
2294	for (i = 0; i < data->vmcount; i++) {
2295	data->vmperms[i].vmid = vmids[i];
2296	data->vmperms[i].perm = QCOM_SCM_PERM_RWX;
2297	}
2298	}
2299
2300	secure_dsp = !(of_property_read_bool(np: rdev->of_node, propname: "qcom,non-secure-domain"));
2301	data->secure = secure_dsp;
2302
2303	switch (domain_id) {
2304	case ADSP_DOMAIN_ID:
2305	case MDSP_DOMAIN_ID:
2306	case SDSP_DOMAIN_ID:
2307	/* Unsigned PD offloading is only supported on CDSP*/
2308	data->unsigned_support = false;
2309	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: secure_dsp, domain: domains[domain_id]);
2310	if (err)
2311	goto fdev_error;
2312	break;
2313	case CDSP_DOMAIN_ID:
2314	data->unsigned_support = true;
2315	/* Create both device nodes so that we can allow both Signed and Unsigned PD */
2316	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: true, domain: domains[domain_id]);
2317	if (err)
2318	goto fdev_error;
2319
2320	err = fastrpc_device_register(dev: rdev, cctx: data, is_secured: false, domain: domains[domain_id]);
2321	if (err)
2322	goto fdev_error;
2323	break;
2324	default:
2325	err = -EINVAL;
2326	goto fdev_error;
2327	}
2328
2329	kref_init(kref: &data->refcount);
2330
2331	dev_set_drvdata(dev: &rpdev->dev, data);
2332	rdev->dma_mask = &data->dma_mask;
2333	dma_set_mask_and_coherent(dev: rdev, DMA_BIT_MASK(32));
2334	INIT_LIST_HEAD(list: &data->users);
2335	INIT_LIST_HEAD(list: &data->invoke_interrupted_mmaps);
2336	spin_lock_init(&data->lock);
2337	idr_init(idr: &data->ctx_idr);
2338	data->domain_id = domain_id;
2339	data->rpdev = rpdev;
2340
2341	err = of_platform_populate(root: rdev->of_node, NULL, NULL, parent: rdev);
2342	if (err)
2343	goto populate_error;
2344
2345	return 0;
2346
2347	populate_error:
2348	if (data->fdevice)
2349	misc_deregister(misc: &data->fdevice->miscdev);
2350	if (data->secure_fdevice)
2351	misc_deregister(misc: &data->secure_fdevice->miscdev);
2352
2353	fdev_error:
2354	kfree(objp: data);
2355	return err;
2356	}
2357
2358	static void fastrpc_notify_users(struct fastrpc_user *user)
2359	{
2360	struct fastrpc_invoke_ctx *ctx;
2361
2362	spin_lock(lock: &user->lock);
2363	list_for_each_entry(ctx, &user->pending, node) {
2364	ctx->retval = -EPIPE;
2365	complete(&ctx->work);
2366	}
2367	spin_unlock(lock: &user->lock);
2368	}
2369
2370	static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)
2371	{
2372	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: &rpdev->dev);
2373	struct fastrpc_buf *buf, *b;
2374	struct fastrpc_user *user;
2375	unsigned long flags;
2376
2377	/* No invocations past this point */
2378	spin_lock_irqsave(&cctx->lock, flags);
2379	cctx->rpdev = NULL;
2380	list_for_each_entry(user, &cctx->users, user)
2381	fastrpc_notify_users(user);
2382	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2383
2384	if (cctx->fdevice)
2385	misc_deregister(misc: &cctx->fdevice->miscdev);
2386
2387	if (cctx->secure_fdevice)
2388	misc_deregister(misc: &cctx->secure_fdevice->miscdev);
2389
2390	list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)
2391	list_del(entry: &buf->node);
2392
2393	if (cctx->remote_heap)
2394	fastrpc_buf_free(buf: cctx->remote_heap);
2395
2396	of_platform_depopulate(parent: &rpdev->dev);
2397
2398	fastrpc_channel_ctx_put(cctx);
2399	}
2400
2401	static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
2402	int len, void *priv, u32 addr)
2403	{
2404	struct fastrpc_channel_ctx *cctx = dev_get_drvdata(dev: &rpdev->dev);
2405	struct fastrpc_invoke_rsp *rsp = data;
2406	struct fastrpc_invoke_ctx *ctx;
2407	unsigned long flags;
2408	unsigned long ctxid;
2409
2410	if (len < sizeof(*rsp))
2411	return -EINVAL;
2412
2413	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
2414
2415	spin_lock_irqsave(&cctx->lock, flags);
2416	ctx = idr_find(&cctx->ctx_idr, id: ctxid);
2417	spin_unlock_irqrestore(lock: &cctx->lock, flags);
2418
2419	if (!ctx) {
2420	dev_err(&rpdev->dev, "No context ID matches response\n");
2421	return -ENOENT;
2422	}
2423
2424	ctx->retval = rsp->retval;
2425	complete(&ctx->work);
2426
2427	/*
2428	* The DMA buffer associated with the context cannot be freed in
2429	* interrupt context so schedule it through a worker thread to
2430	* avoid a kernel BUG.
2431	*/
2432	schedule_work(work: &ctx->put_work);
2433
2434	return 0;
2435	}
2436
2437	static const struct of_device_id fastrpc_rpmsg_of_match[] = {
2438	{ .compatible = "qcom,fastrpc" },
2439	{ },
2440	};
2441	MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);
2442
2443	static struct rpmsg_driver fastrpc_driver = {
2444	.probe = fastrpc_rpmsg_probe,
2445	.remove = fastrpc_rpmsg_remove,
2446	.callback = fastrpc_rpmsg_callback,
2447	.drv = {
2448	.name = "qcom,fastrpc",
2449	.of_match_table = fastrpc_rpmsg_of_match,
2450	},
2451	};
2452
2453	static int fastrpc_init(void)
2454	{
2455	int ret;
2456
2457	ret = platform_driver_register(&fastrpc_cb_driver);
2458	if (ret < 0) {
2459	pr_err("fastrpc: failed to register cb driver\n");
2460	return ret;
2461	}
2462
2463	ret = register_rpmsg_driver(&fastrpc_driver);
2464	if (ret < 0) {
2465	pr_err("fastrpc: failed to register rpmsg driver\n");
2466	platform_driver_unregister(&fastrpc_cb_driver);
2467	return ret;
2468	}
2469
2470	return 0;
2471	}
2472	module_init(fastrpc_init);
2473
2474	static void fastrpc_exit(void)
2475	{
2476	platform_driver_unregister(&fastrpc_cb_driver);
2477	unregister_rpmsg_driver(drv: &fastrpc_driver);
2478	}
2479	module_exit(fastrpc_exit);
2480
2481	MODULE_LICENSE("GPL v2");
2482	MODULE_IMPORT_NS(DMA_BUF);
2483