1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (c) 2015-2021, 2023 Linaro Limited
4	* Copyright (c) 2016, EPAM Systems
5	*/
6
7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9	#include <linux/arm-smccc.h>
10	#include <linux/cpuhotplug.h>
11	#include <linux/errno.h>
12	#include <linux/firmware.h>
13	#include <linux/interrupt.h>
14	#include <linux/io.h>
15	#include <linux/irqdomain.h>
16	#include <linux/kernel.h>
17	#include <linux/mm.h>
18	#include <linux/module.h>
19	#include <linux/of.h>
20	#include <linux/of_irq.h>
21	#include <linux/of_platform.h>
22	#include <linux/platform_device.h>
23	#include <linux/rpmb.h>
24	#include <linux/sched.h>
25	#include <linux/slab.h>
26	#include <linux/string.h>
27	#include <linux/tee_core.h>
28	#include <linux/types.h>
29	#include <linux/workqueue.h>
30	#include "optee_private.h"
31	#include "optee_smc.h"
32	#include "optee_rpc_cmd.h"
33	#include <linux/kmemleak.h>
34	#define CREATE_TRACE_POINTS
35	#include "optee_trace.h"
36
37	/*
38	* This file implement the SMC ABI used when communicating with secure world
39	* OP-TEE OS via raw SMCs.
40	* This file is divided into the following sections:
41	* 1. Convert between struct tee_param and struct optee_msg_param
42	* 2. Low level support functions to register shared memory in secure world
43	* 3. Dynamic shared memory pool based on alloc_pages()
44	* 4. Do a normal scheduled call into secure world
45	* 5. Asynchronous notification
46	* 6. Driver initialization.
47	*/
48
49	/*
50	* A typical OP-TEE private shm allocation is 224 bytes (argument struct
51	* with 6 parameters, needed for open session). So with an alignment of 512
52	* we'll waste a bit more than 50%. However, it's only expected that we'll
53	* have a handful of these structs allocated at a time. Most memory will
54	* be allocated aligned to the page size, So all in all this should scale
55	* up and down quite well.
56	*/
57	#define OPTEE_MIN_STATIC_POOL_ALIGN 9 /* 512 bytes aligned */
58
59	/* SMC ABI considers at most a single TEE firmware */
60	static unsigned int pcpu_irq_num;
61
62	static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
63	{
64	enable_percpu_irq(irq: pcpu_irq_num, type: IRQ_TYPE_NONE);
65
66	return 0;
67	}
68
69	static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
70	{
71	disable_percpu_irq(irq: pcpu_irq_num);
72
73	return 0;
74	}
75
76	/*
77	* 1. Convert between struct tee_param and struct optee_msg_param
78	*
79	* optee_from_msg_param() and optee_to_msg_param() are the main
80	* functions.
81	*/
82
83	static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
84	const struct optee_msg_param *mp)
85	{
86	struct tee_shm *shm;
87	phys_addr_t pa;
88	int rc;
89
90	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
91	attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
92	p->u.memref.size = mp->u.tmem.size;
93	shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
94	if (!shm) {
95	p->u.memref.shm_offs = 0;
96	p->u.memref.shm = NULL;
97	return 0;
98	}
99
100	rc = tee_shm_get_pa(shm, offs: 0, pa: &pa);
101	if (rc)
102	return rc;
103
104	p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
105	p->u.memref.shm = shm;
106
107	return 0;
108	}
109
110	static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
111	const struct optee_msg_param *mp)
112	{
113	struct tee_shm *shm;
114
115	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
116	attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
117	p->u.memref.size = mp->u.rmem.size;
118	shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
119
120	if (shm) {
121	p->u.memref.shm_offs = mp->u.rmem.offs;
122	p->u.memref.shm = shm;
123	} else {
124	p->u.memref.shm_offs = 0;
125	p->u.memref.shm = NULL;
126	}
127	}
128
129	/**
130	* optee_from_msg_param() - convert from OPTEE_MSG parameters to
131	* struct tee_param
132	* @optee: main service struct
133	* @params: subsystem internal parameter representation
134	* @num_params: number of elements in the parameter arrays
135	* @msg_params: OPTEE_MSG parameters
136	* Returns 0 on success or <0 on failure
137	*/
138	static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
139	size_t num_params,
140	const struct optee_msg_param *msg_params)
141	{
142	int rc;
143	size_t n;
144
145	for (n = 0; n < num_params; n++) {
146	struct tee_param *p = params + n;
147	const struct optee_msg_param *mp = msg_params + n;
148	u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
149
150	switch (attr) {
151	case OPTEE_MSG_ATTR_TYPE_NONE:
152	p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
153	memset(&p->u, 0, sizeof(p->u));
154	break;
155	case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
156	case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
157	case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
158	optee_from_msg_param_value(p, attr, mp);
159	break;
160	case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
161	case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
162	case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
163	rc = from_msg_param_tmp_mem(p, attr, mp);
164	if (rc)
165	return rc;
166	break;
167	case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
168	case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
169	case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
170	from_msg_param_reg_mem(p, attr, mp);
171	break;
172
173	default:
174	return -EINVAL;
175	}
176	}
177	return 0;
178	}
179
180	static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
181	const struct tee_param *p)
182	{
183	int rc;
184	phys_addr_t pa;
185
186	mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
187	TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
188
189	mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
190	mp->u.tmem.size = p->u.memref.size;
191
192	if (!p->u.memref.shm) {
193	mp->u.tmem.buf_ptr = 0;
194	return 0;
195	}
196
197	rc = tee_shm_get_pa(shm: p->u.memref.shm, offs: p->u.memref.shm_offs, pa: &pa);
198	if (rc)
199	return rc;
200
201	mp->u.tmem.buf_ptr = pa;
202	mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
203	OPTEE_MSG_ATTR_CACHE_SHIFT;
204
205	return 0;
206	}
207
208	static int to_msg_param_reg_mem(struct optee_msg_param *mp,
209	const struct tee_param *p)
210	{
211	mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
212	TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
213
214	mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
215	mp->u.rmem.size = p->u.memref.size;
216	mp->u.rmem.offs = p->u.memref.shm_offs;
217	return 0;
218	}
219
220	/**
221	* optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
222	* @optee: main service struct
223	* @msg_params: OPTEE_MSG parameters
224	* @num_params: number of elements in the parameter arrays
225	* @params: subsystem itnernal parameter representation
226	* Returns 0 on success or <0 on failure
227	*/
228	static int optee_to_msg_param(struct optee *optee,
229	struct optee_msg_param *msg_params,
230	size_t num_params, const struct tee_param *params)
231	{
232	int rc;
233	size_t n;
234
235	for (n = 0; n < num_params; n++) {
236	const struct tee_param *p = params + n;
237	struct optee_msg_param *mp = msg_params + n;
238
239	switch (p->attr) {
240	case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
241	mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
242	memset(&mp->u, 0, sizeof(mp->u));
243	break;
244	case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
245	case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
246	case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
247	optee_to_msg_param_value(mp, p);
248	break;
249	case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
250	case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
251	case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
252	if (tee_shm_is_dynamic(shm: p->u.memref.shm))
253	rc = to_msg_param_reg_mem(mp, p);
254	else
255	rc = to_msg_param_tmp_mem(mp, p);
256	if (rc)
257	return rc;
258	break;
259	default:
260	return -EINVAL;
261	}
262	}
263	return 0;
264	}
265
266	/*
267	* 2. Low level support functions to register shared memory in secure world
268	*
269	* Functions to enable/disable shared memory caching in secure world, that
270	* is, lazy freeing of previously allocated shared memory. Freeing is
271	* performed when a request has been compled.
272	*
273	* Functions to register and unregister shared memory both for normal
274	* clients and for tee-supplicant.
275	*/
276
277	/**
278	* optee_enable_shm_cache() - Enables caching of some shared memory allocation
279	* in OP-TEE
280	* @optee: main service struct
281	*/
282	static void optee_enable_shm_cache(struct optee *optee)
283	{
284	struct optee_call_waiter w;
285
286	/* We need to retry until secure world isn't busy. */
287	optee_cq_wait_init(cq: &optee->call_queue, w: &w, sys_thread: false);
288	while (true) {
289	struct arm_smccc_res res;
290
291	optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
292	0, 0, 0, 0, 0, 0, 0, &res);
293	if (res.a0 == OPTEE_SMC_RETURN_OK)
294	break;
295	optee_cq_wait_for_completion(cq: &optee->call_queue, w: &w);
296	}
297	optee_cq_wait_final(cq: &optee->call_queue, w: &w);
298	}
299
300	/**
301	* __optee_disable_shm_cache() - Disables caching of some shared memory
302	* allocation in OP-TEE
303	* @optee: main service struct
304	* @is_mapped: true if the cached shared memory addresses were mapped by this
305	* kernel, are safe to dereference, and should be freed
306	*/
307	static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
308	{
309	struct optee_call_waiter w;
310
311	/* We need to retry until secure world isn't busy. */
312	optee_cq_wait_init(cq: &optee->call_queue, w: &w, sys_thread: false);
313	while (true) {
314	union {
315	struct arm_smccc_res smccc;
316	struct optee_smc_disable_shm_cache_result result;
317	} res;
318
319	optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
320	0, 0, 0, 0, 0, 0, 0, &res.smccc);
321	if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
322	break; /* All shm's freed */
323	if (res.result.status == OPTEE_SMC_RETURN_OK) {
324	struct tee_shm *shm;
325
326	/*
327	* Shared memory references that were not mapped by
328	* this kernel must be ignored to prevent a crash.
329	*/
330	if (!is_mapped)
331	continue;
332
333	shm = reg_pair_to_ptr(reg0: res.result.shm_upper32,
334	reg1: res.result.shm_lower32);
335	tee_shm_free(shm);
336	} else {
337	optee_cq_wait_for_completion(cq: &optee->call_queue, w: &w);
338	}
339	}
340	optee_cq_wait_final(cq: &optee->call_queue, w: &w);
341	}
342
343	/**
344	* optee_disable_shm_cache() - Disables caching of mapped shared memory
345	* allocations in OP-TEE
346	* @optee: main service struct
347	*/
348	static void optee_disable_shm_cache(struct optee *optee)
349	{
350	return __optee_disable_shm_cache(optee, is_mapped: true);
351	}
352
353	/**
354	* optee_disable_unmapped_shm_cache() - Disables caching of shared memory
355	* allocations in OP-TEE which are not
356	* currently mapped
357	* @optee: main service struct
358	*/
359	static void optee_disable_unmapped_shm_cache(struct optee *optee)
360	{
361	return __optee_disable_shm_cache(optee, is_mapped: false);
362	}
363
364	#define PAGELIST_ENTRIES_PER_PAGE \
365	((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
366
367	/*
368	* The final entry in each pagelist page is a pointer to the next
369	* pagelist page.
370	*/
371	static size_t get_pages_list_size(size_t num_entries)
372	{
373	int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
374
375	return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
376	}
377
378	static u64 *optee_allocate_pages_list(size_t num_entries)
379	{
380	return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
381	}
382
383	static void optee_free_pages_list(void *list, size_t num_entries)
384	{
385	free_pages_exact(virt: list, size: get_pages_list_size(num_entries));
386	}
387
388	/**
389	* optee_fill_pages_list() - write list of user pages to given shared
390	* buffer.
391	*
392	* @dst: page-aligned buffer where list of pages will be stored
393	* @pages: array of pages that represents shared buffer
394	* @num_pages: number of entries in @pages
395	* @page_offset: offset of user buffer from page start
396	*
397	* @dst should be big enough to hold list of user page addresses and
398	* links to the next pages of buffer
399	*/
400	static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
401	size_t page_offset)
402	{
403	int n = 0;
404	phys_addr_t optee_page;
405	/*
406	* Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
407	* for details.
408	*/
409	struct {
410	u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
411	u64 next_page_data;
412	} *pages_data;
413
414	/*
415	* Currently OP-TEE uses 4k page size and it does not looks
416	* like this will change in the future. On other hand, there are
417	* no know ARM architectures with page size < 4k.
418	* Thus the next built assert looks redundant. But the following
419	* code heavily relies on this assumption, so it is better be
420	* safe than sorry.
421	*/
422	BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
423
424	pages_data = (void *)dst;
425	/*
426	* If linux page is bigger than 4k, and user buffer offset is
427	* larger than 4k/8k/12k/etc this will skip first 4k pages,
428	* because they bear no value data for OP-TEE.
429	*/
430	optee_page = page_to_phys(*pages) +
431	round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
432
433	while (true) {
434	pages_data->pages_list[n++] = optee_page;
435
436	if (n == PAGELIST_ENTRIES_PER_PAGE) {
437	pages_data->next_page_data =
438	virt_to_phys(address: pages_data + 1);
439	pages_data++;
440	n = 0;
441	}
442
443	optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
444	if (!(optee_page & ~PAGE_MASK)) {
445	if (!--num_pages)
446	break;
447	pages++;
448	optee_page = page_to_phys(*pages);
449	}
450	}
451	}
452
453	static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
454	struct page **pages, size_t num_pages,
455	unsigned long start)
456	{
457	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
458	struct optee_msg_arg *msg_arg;
459	struct tee_shm *shm_arg;
460	u64 *pages_list;
461	size_t sz;
462	int rc;
463
464	if (!num_pages)
465	return -EINVAL;
466
467	rc = optee_check_mem_type(start, num_pages);
468	if (rc)
469	return rc;
470
471	pages_list = optee_allocate_pages_list(num_entries: num_pages);
472	if (!pages_list)
473	return -ENOMEM;
474
475	/*
476	* We're about to register shared memory we can't register shared
477	* memory for this request or there's a catch-22.
478	*
479	* So in this we'll have to do the good old temporary private
480	* allocation instead of using optee_get_msg_arg().
481	*/
482	sz = optee_msg_arg_size(rpc_param_count: optee->rpc_param_count);
483	shm_arg = tee_shm_alloc_priv_buf(ctx, size: sz);
484	if (IS_ERR(ptr: shm_arg)) {
485	rc = PTR_ERR(ptr: shm_arg);
486	goto out;
487	}
488	msg_arg = tee_shm_get_va(shm: shm_arg, offs: 0);
489	if (IS_ERR(ptr: msg_arg)) {
490	rc = PTR_ERR(ptr: msg_arg);
491	goto out;
492	}
493
494	optee_fill_pages_list(dst: pages_list, pages, num_pages,
495	page_offset: tee_shm_get_page_offset(shm));
496
497	memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
498	msg_arg->num_params = 1;
499	msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
500	msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
501	OPTEE_MSG_ATTR_NONCONTIG;
502	msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
503	msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
504	/*
505	* In the least bits of msg_arg->params->u.tmem.buf_ptr we
506	* store buffer offset from 4k page, as described in OP-TEE ABI.
507	*/
508	msg_arg->params->u.tmem.buf_ptr = virt_to_phys(address: pages_list) |
509	(tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
510
511	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
512	msg_arg->ret != TEEC_SUCCESS)
513	rc = -EINVAL;
514
515	tee_shm_free(shm: shm_arg);
516	out:
517	optee_free_pages_list(list: pages_list, num_entries: num_pages);
518	return rc;
519	}
520
521	static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
522	{
523	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
524	struct optee_msg_arg *msg_arg;
525	struct tee_shm *shm_arg;
526	int rc = 0;
527	size_t sz;
528
529	/*
530	* We're about to unregister shared memory and we may not be able
531	* register shared memory for this request in case we're called
532	* from optee_shm_arg_cache_uninit().
533	*
534	* So in order to keep things simple in this function just as in
535	* optee_shm_register() we'll use temporary private allocation
536	* instead of using optee_get_msg_arg().
537	*/
538	sz = optee_msg_arg_size(rpc_param_count: optee->rpc_param_count);
539	shm_arg = tee_shm_alloc_priv_buf(ctx, size: sz);
540	if (IS_ERR(ptr: shm_arg))
541	return PTR_ERR(ptr: shm_arg);
542	msg_arg = tee_shm_get_va(shm: shm_arg, offs: 0);
543	if (IS_ERR(ptr: msg_arg)) {
544	rc = PTR_ERR(ptr: msg_arg);
545	goto out;
546	}
547
548	memset(msg_arg, 0, sz);
549	msg_arg->num_params = 1;
550	msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
551	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
552	msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
553
554	if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) ||
555	msg_arg->ret != TEEC_SUCCESS)
556	rc = -EINVAL;
557	out:
558	tee_shm_free(shm: shm_arg);
559	return rc;
560	}
561
562	static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
563	struct page **pages, size_t num_pages,
564	unsigned long start)
565	{
566	/*
567	* We don't want to register supplicant memory in OP-TEE.
568	* Instead information about it will be passed in RPC code.
569	*/
570	return optee_check_mem_type(start, num_pages);
571	}
572
573	static int optee_shm_unregister_supp(struct tee_context *ctx,
574	struct tee_shm *shm)
575	{
576	return 0;
577	}
578
579	/*
580	* 3. Dynamic shared memory pool based on alloc_pages()
581	*
582	* Implements an OP-TEE specific shared memory pool which is used
583	* when dynamic shared memory is supported by secure world.
584	*
585	* The main function is optee_shm_pool_alloc_pages().
586	*/
587
588	static int pool_op_alloc(struct tee_shm_pool *pool,
589	struct tee_shm *shm, size_t size, size_t align)
590	{
591	/*
592	* Shared memory private to the OP-TEE driver doesn't need
593	* to be registered with OP-TEE.
594	*/
595	if (shm->flags & TEE_SHM_PRIV)
596	return tee_dyn_shm_alloc_helper(shm, size, align, NULL);
597
598	return tee_dyn_shm_alloc_helper(shm, size, align, shm_register: optee_shm_register);
599	}
600
601	static void pool_op_free(struct tee_shm_pool *pool,
602	struct tee_shm *shm)
603	{
604	if (!(shm->flags & TEE_SHM_PRIV))
605	tee_dyn_shm_free_helper(shm, shm_unregister: optee_shm_unregister);
606	else
607	tee_dyn_shm_free_helper(shm, NULL);
608	}
609
610	static void pool_op_destroy_pool(struct tee_shm_pool *pool)
611	{
612	kfree(objp: pool);
613	}
614
615	static const struct tee_shm_pool_ops pool_ops = {
616	.alloc = pool_op_alloc,
617	.free = pool_op_free,
618	.destroy_pool = pool_op_destroy_pool,
619	};
620
621	/**
622	* optee_shm_pool_alloc_pages() - create page-based allocator pool
623	*
624	* This pool is used when OP-TEE supports dymanic SHM. In this case
625	* command buffers and such are allocated from kernel's own memory.
626	*/
627	static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
628	{
629	struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
630
631	if (!pool)
632	return ERR_PTR(error: -ENOMEM);
633
634	pool->ops = &pool_ops;
635
636	return pool;
637	}
638
639	/*
640	* 4. Do a normal scheduled call into secure world
641	*
642	* The function optee_smc_do_call_with_arg() performs a normal scheduled
643	* call into secure world. During this call may normal world request help
644	* from normal world using RPCs, Remote Procedure Calls. This includes
645	* delivery of non-secure interrupts to for instance allow rescheduling of
646	* the current task.
647	*/
648
649	static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
650	struct optee_msg_arg *arg)
651	{
652	struct tee_shm *shm;
653
654	arg->ret_origin = TEEC_ORIGIN_COMMS;
655
656	if (arg->num_params != 1 ||
657	arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
658	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
659	return;
660	}
661
662	shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
663	switch (arg->params[0].u.value.a) {
664	case OPTEE_RPC_SHM_TYPE_APPL:
665	optee_rpc_cmd_free_suppl(ctx, shm);
666	break;
667	case OPTEE_RPC_SHM_TYPE_KERNEL:
668	tee_shm_free(shm);
669	break;
670	default:
671	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
672	}
673	arg->ret = TEEC_SUCCESS;
674	}
675
676	static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
677	struct optee *optee,
678	struct optee_msg_arg *arg,
679	struct optee_call_ctx *call_ctx)
680	{
681	struct tee_shm *shm;
682	size_t sz;
683	size_t n;
684	struct page **pages;
685	size_t page_count;
686
687	arg->ret_origin = TEEC_ORIGIN_COMMS;
688
689	if (!arg->num_params ||
690	arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
691	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
692	return;
693	}
694
695	for (n = 1; n < arg->num_params; n++) {
696	if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
697	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
698	return;
699	}
700	}
701
702	sz = arg->params[0].u.value.b;
703	switch (arg->params[0].u.value.a) {
704	case OPTEE_RPC_SHM_TYPE_APPL:
705	shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
706	break;
707	case OPTEE_RPC_SHM_TYPE_KERNEL:
708	shm = tee_shm_alloc_priv_buf(ctx: optee->ctx, size: sz);
709	break;
710	default:
711	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
712	return;
713	}
714
715	if (IS_ERR(ptr: shm)) {
716	arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
717	return;
718	}
719
720	/*
721	* If there are pages it's dynamically allocated shared memory (not
722	* from the reserved shared memory pool) and needs to be
723	* registered.
724	*/
725	pages = tee_shm_get_pages(shm, num_pages: &page_count);
726	if (pages) {
727	u64 *pages_list;
728
729	pages_list = optee_allocate_pages_list(num_entries: page_count);
730	if (!pages_list) {
731	arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
732	goto bad;
733	}
734
735	call_ctx->pages_list = pages_list;
736	call_ctx->num_entries = page_count;
737
738	arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
739	OPTEE_MSG_ATTR_NONCONTIG;
740	/*
741	* In the least bits of u.tmem.buf_ptr we store buffer offset
742	* from 4k page, as described in OP-TEE ABI.
743	*/
744	arg->params[0].u.tmem.buf_ptr = virt_to_phys(address: pages_list) |
745	(tee_shm_get_page_offset(shm) &
746	(OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
747
748	optee_fill_pages_list(dst: pages_list, pages, num_pages: page_count,
749	page_offset: tee_shm_get_page_offset(shm));
750	} else {
751	phys_addr_t pa;
752
753	if (tee_shm_get_pa(shm, offs: 0, pa: &pa)) {
754	arg->ret = TEEC_ERROR_BAD_PARAMETERS;
755	goto bad;
756	}
757
758	arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
759	arg->params[0].u.tmem.buf_ptr = pa;
760	}
761	arg->params[0].u.tmem.size = tee_shm_get_size(shm);
762	arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
763
764	arg->ret = TEEC_SUCCESS;
765	return;
766	bad:
767	tee_shm_free(shm);
768	}
769
770	static void free_pages_list(struct optee_call_ctx *call_ctx)
771	{
772	if (call_ctx->pages_list) {
773	optee_free_pages_list(list: call_ctx->pages_list,
774	num_entries: call_ctx->num_entries);
775	call_ctx->pages_list = NULL;
776	call_ctx->num_entries = 0;
777	}
778	}
779
780	static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
781	{
782	free_pages_list(call_ctx);
783	}
784
785	static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
786	struct optee_msg_arg *arg,
787	struct optee_call_ctx *call_ctx)
788	{
789
790	switch (arg->cmd) {
791	case OPTEE_RPC_CMD_SHM_ALLOC:
792	free_pages_list(call_ctx);
793	handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
794	break;
795	case OPTEE_RPC_CMD_SHM_FREE:
796	handle_rpc_func_cmd_shm_free(ctx, arg);
797	break;
798	default:
799	optee_rpc_cmd(ctx, optee, arg);
800	}
801	}
802
803	/**
804	* optee_handle_rpc() - handle RPC from secure world
805	* @ctx: context doing the RPC
806	* @rpc_arg: pointer to RPC arguments if any, or NULL if none
807	* @param: value of registers for the RPC
808	* @call_ctx: call context. Preserved during one OP-TEE invocation
809	*
810	* Result of RPC is written back into @param.
811	*/
812	static void optee_handle_rpc(struct tee_context *ctx,
813	struct optee_msg_arg *rpc_arg,
814	struct optee_rpc_param *param,
815	struct optee_call_ctx *call_ctx)
816	{
817	struct tee_device *teedev = ctx->teedev;
818	struct optee *optee = tee_get_drvdata(teedev);
819	struct optee_msg_arg *arg;
820	struct tee_shm *shm;
821	phys_addr_t pa;
822
823	switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
824	case OPTEE_SMC_RPC_FUNC_ALLOC:
825	shm = tee_shm_alloc_priv_buf(ctx: optee->ctx, size: param->a1);
826	if (!IS_ERR(ptr: shm) && !tee_shm_get_pa(shm, offs: 0, pa: &pa)) {
827	reg_pair_from_64(reg0: &param->a1, reg1: &param->a2, val: pa);
828	reg_pair_from_64(reg0: &param->a4, reg1: &param->a5,
829	val: (unsigned long)shm);
830	} else {
831	param->a1 = 0;
832	param->a2 = 0;
833	param->a4 = 0;
834	param->a5 = 0;
835	}
836	kmemleak_not_leak(ptr: shm);
837	break;
838	case OPTEE_SMC_RPC_FUNC_FREE:
839	shm = reg_pair_to_ptr(reg0: param->a1, reg1: param->a2);
840	tee_shm_free(shm);
841	break;
842	case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
843	/*
844	* A foreign interrupt was raised while secure world was
845	* executing, since they are handled in Linux a dummy RPC is
846	* performed to let Linux take the interrupt through the normal
847	* vector.
848	*/
849	break;
850	case OPTEE_SMC_RPC_FUNC_CMD:
851	if (rpc_arg) {
852	arg = rpc_arg;
853	} else {
854	shm = reg_pair_to_ptr(reg0: param->a1, reg1: param->a2);
855	arg = tee_shm_get_va(shm, offs: 0);
856	if (IS_ERR(ptr: arg)) {
857	pr_err("%s: tee_shm_get_va %p failed\n",
858	__func__, shm);
859	break;
860	}
861	}
862
863	handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
864	break;
865	default:
866	pr_warn("Unknown RPC func 0x%x\n",
867	(u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
868	break;
869	}
870
871	param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
872	}
873
874	/**
875	* optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
876	* @ctx: calling context
877	* @shm: shared memory holding the message to pass to secure world
878	* @offs: offset of the message in @shm
879	* @system_thread: true if caller requests TEE system thread support
880	*
881	* Does and SMC to OP-TEE in secure world and handles eventual resulting
882	* Remote Procedure Calls (RPC) from OP-TEE.
883	*
884	* Returns return code from secure world, 0 is OK
885	*/
886	static int optee_smc_do_call_with_arg(struct tee_context *ctx,
887	struct tee_shm *shm, u_int offs,
888	bool system_thread)
889	{
890	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
891	struct optee_call_waiter w;
892	struct optee_rpc_param param = { };
893	struct optee_call_ctx call_ctx = { };
894	struct optee_msg_arg *rpc_arg = NULL;
895	int rc;
896
897	if (optee->rpc_param_count) {
898	struct optee_msg_arg *arg;
899	unsigned int rpc_arg_offs;
900
901	arg = tee_shm_get_va(shm, offs);
902	if (IS_ERR(ptr: arg))
903	return PTR_ERR(ptr: arg);
904
905	rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
906	rpc_arg = tee_shm_get_va(shm, offs: offs + rpc_arg_offs);
907	if (IS_ERR(ptr: rpc_arg))
908	return PTR_ERR(ptr: rpc_arg);
909	}
910
911	if (rpc_arg && tee_shm_is_dynamic(shm)) {
912	param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
913	reg_pair_from_64(reg0: &param.a1, reg1: &param.a2, val: (u_long)shm);
914	param.a3 = offs;
915	} else {
916	phys_addr_t parg;
917
918	rc = tee_shm_get_pa(shm, offs, pa: &parg);
919	if (rc)
920	return rc;
921
922	if (rpc_arg)
923	param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
924	else
925	param.a0 = OPTEE_SMC_CALL_WITH_ARG;
926	reg_pair_from_64(reg0: &param.a1, reg1: &param.a2, val: parg);
927	}
928	/* Initialize waiter */
929	optee_cq_wait_init(cq: &optee->call_queue, w: &w, sys_thread: system_thread);
930	while (true) {
931	struct arm_smccc_res res;
932
933	trace_optee_invoke_fn_begin(param: &param);
934	optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
935	param.a4, param.a5, param.a6, param.a7,
936	&res);
937	trace_optee_invoke_fn_end(param: &param, res: &res);
938
939	if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
940	/*
941	* Out of threads in secure world, wait for a thread
942	* become available.
943	*/
944	optee_cq_wait_for_completion(cq: &optee->call_queue, w: &w);
945	} else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
946	cond_resched();
947	param.a0 = res.a0;
948	param.a1 = res.a1;
949	param.a2 = res.a2;
950	param.a3 = res.a3;
951	optee_handle_rpc(ctx, rpc_arg, param: &param, call_ctx: &call_ctx);
952	} else {
953	rc = res.a0;
954	break;
955	}
956	}
957
958	optee_rpc_finalize_call(call_ctx: &call_ctx);
959	/*
960	* We're done with our thread in secure world, if there's any
961	* thread waiters wake up one.
962	*/
963	optee_cq_wait_final(cq: &optee->call_queue, w: &w);
964
965	return rc;
966	}
967
968	static int optee_smc_lend_protmem(struct optee *optee, struct tee_shm *protmem,
969	u32 *mem_attrs, unsigned int ma_count,
970	u32 use_case)
971	{
972	struct optee_shm_arg_entry *entry;
973	struct optee_msg_arg *msg_arg;
974	struct tee_shm *shm;
975	u_int offs;
976	int rc;
977
978	msg_arg = optee_get_msg_arg(ctx: optee->ctx, num_params: 2, entry: &entry, shm_ret: &shm, offs: &offs);
979	if (IS_ERR(ptr: msg_arg))
980	return PTR_ERR(ptr: msg_arg);
981
982	msg_arg->cmd = OPTEE_MSG_CMD_LEND_PROTMEM;
983	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT;
984	msg_arg->params[0].u.value.a = use_case;
985	msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
986	msg_arg->params[1].u.tmem.buf_ptr = protmem->paddr;
987	msg_arg->params[1].u.tmem.size = protmem->size;
988	msg_arg->params[1].u.tmem.shm_ref = (u_long)protmem;
989
990	rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false);
991	if (rc)
992	goto out;
993	if (msg_arg->ret != TEEC_SUCCESS) {
994	rc = -EINVAL;
995	goto out;
996	}
997	protmem->sec_world_id = (u_long)protmem;
998
999	out:
1000	optee_free_msg_arg(ctx: optee->ctx, entry, offs);
1001	return rc;
1002	}
1003
1004	static int optee_smc_reclaim_protmem(struct optee *optee,
1005	struct tee_shm *protmem)
1006	{
1007	struct optee_shm_arg_entry *entry;
1008	struct optee_msg_arg *msg_arg;
1009	struct tee_shm *shm;
1010	u_int offs;
1011	int rc;
1012
1013	msg_arg = optee_get_msg_arg(ctx: optee->ctx, num_params: 1, entry: &entry, shm_ret: &shm, offs: &offs);
1014	if (IS_ERR(ptr: msg_arg))
1015	return PTR_ERR(ptr: msg_arg);
1016
1017	msg_arg->cmd = OPTEE_MSG_CMD_RECLAIM_PROTMEM;
1018	msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
1019	msg_arg->params[0].u.rmem.shm_ref = (u_long)protmem;
1020
1021	rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false);
1022	if (rc)
1023	goto out;
1024	if (msg_arg->ret != TEEC_SUCCESS)
1025	rc = -EINVAL;
1026
1027	out:
1028	optee_free_msg_arg(ctx: optee->ctx, entry, offs);
1029	return rc;
1030	}
1031
1032	/*
1033	* 5. Asynchronous notification
1034	*/
1035
1036	static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
1037	bool *value_pending)
1038	{
1039	struct arm_smccc_res res;
1040
1041	invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
1042
1043	if (res.a0) {
1044	*value_valid = false;
1045	return 0;
1046	}
1047	*value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
1048	*value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
1049	return res.a1;
1050	}
1051
1052	static irqreturn_t irq_handler(struct optee *optee)
1053	{
1054	bool do_bottom_half = false;
1055	bool value_valid;
1056	bool value_pending;
1057	u32 value;
1058
1059	do {
1060	value = get_async_notif_value(invoke_fn: optee->smc.invoke_fn,
1061	value_valid: &value_valid, value_pending: &value_pending);
1062	if (!value_valid)
1063	break;
1064
1065	if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
1066	do_bottom_half = true;
1067	else
1068	optee_notif_send(optee, key: value);
1069	} while (value_pending);
1070
1071	if (do_bottom_half)
1072	return IRQ_WAKE_THREAD;
1073	return IRQ_HANDLED;
1074	}
1075
1076	static irqreturn_t notif_irq_handler(int irq, void *dev_id)
1077	{
1078	struct optee *optee = dev_id;
1079
1080	return irq_handler(optee);
1081	}
1082
1083	static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
1084	{
1085	struct optee *optee = dev_id;
1086
1087	optee_do_bottom_half(ctx: optee->ctx);
1088
1089	return IRQ_HANDLED;
1090	}
1091
1092	static int init_irq(struct optee *optee, u_int irq)
1093	{
1094	int rc;
1095
1096	rc = request_threaded_irq(irq, handler: notif_irq_handler,
1097	thread_fn: notif_irq_thread_fn,
1098	flags: 0, name: "optee_notification", dev: optee);
1099	if (rc)
1100	return rc;
1101
1102	optee->smc.notif_irq = irq;
1103
1104	return 0;
1105	}
1106
1107	static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
1108	{
1109	struct optee_pcpu *pcpu = dev_id;
1110	struct optee *optee = pcpu->optee;
1111
1112	if (irq_handler(optee) == IRQ_WAKE_THREAD)
1113	queue_work(wq: optee->smc.notif_pcpu_wq,
1114	work: &optee->smc.notif_pcpu_work);
1115
1116	return IRQ_HANDLED;
1117	}
1118
1119	static void notif_pcpu_irq_work_fn(struct work_struct *work)
1120	{
1121	struct optee_smc *optee_smc = container_of(work, struct optee_smc,
1122	notif_pcpu_work);
1123	struct optee *optee = container_of(optee_smc, struct optee, smc);
1124
1125	optee_do_bottom_half(ctx: optee->ctx);
1126	}
1127
1128	static int init_pcpu_irq(struct optee *optee, u_int irq)
1129	{
1130	struct optee_pcpu __percpu *optee_pcpu;
1131	int cpu, rc;
1132
1133	optee_pcpu = alloc_percpu(struct optee_pcpu);
1134	if (!optee_pcpu)
1135	return -ENOMEM;
1136
1137	for_each_present_cpu(cpu)
1138	per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
1139
1140	rc = request_percpu_irq(irq, handler: notif_pcpu_irq_handler,
1141	devname: "optee_pcpu_notification", percpu_dev_id: optee_pcpu);
1142	if (rc)
1143	goto err_free_pcpu;
1144
1145	INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
1146	optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
1147	if (!optee->smc.notif_pcpu_wq) {
1148	rc = -EINVAL;
1149	goto err_free_pcpu_irq;
1150	}
1151
1152	optee->smc.optee_pcpu = optee_pcpu;
1153	optee->smc.notif_irq = irq;
1154
1155	pcpu_irq_num = irq;
1156	rc = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "optee/pcpu-notif:starting",
1157	startup: optee_cpuhp_enable_pcpu_irq,
1158	teardown: optee_cpuhp_disable_pcpu_irq);
1159	if (!rc)
1160	rc = -EINVAL;
1161	if (rc < 0)
1162	goto err_free_pcpu_irq;
1163
1164	optee->smc.notif_cpuhp_state = rc;
1165
1166	return 0;
1167
1168	err_free_pcpu_irq:
1169	free_percpu_irq(irq, optee_pcpu);
1170	err_free_pcpu:
1171	free_percpu(pdata: optee_pcpu);
1172
1173	return rc;
1174	}
1175
1176	static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
1177	{
1178	if (irq_is_percpu_devid(irq))
1179	return init_pcpu_irq(optee, irq);
1180	else
1181	return init_irq(optee, irq);
1182	}
1183
1184	static void uninit_pcpu_irq(struct optee *optee)
1185	{
1186	cpuhp_remove_state(state: optee->smc.notif_cpuhp_state);
1187
1188	destroy_workqueue(wq: optee->smc.notif_pcpu_wq);
1189
1190	free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
1191	free_percpu(pdata: optee->smc.optee_pcpu);
1192	}
1193
1194	static void optee_smc_notif_uninit_irq(struct optee *optee)
1195	{
1196	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1197	optee_stop_async_notif(ctx: optee->ctx);
1198	if (optee->smc.notif_irq) {
1199	if (irq_is_percpu_devid(irq: optee->smc.notif_irq))
1200	uninit_pcpu_irq(optee);
1201	else
1202	free_irq(optee->smc.notif_irq, optee);
1203
1204	irq_dispose_mapping(virq: optee->smc.notif_irq);
1205	}
1206	}
1207	}
1208
1209	/*
1210	* 6. Driver initialization
1211	*
1212	* During driver initialization is secure world probed to find out which
1213	* features it supports so the driver can be initialized with a matching
1214	* configuration. This involves for instance support for dynamic shared
1215	* memory instead of a static memory carvout.
1216	*/
1217
1218	static void optee_get_version(struct tee_device *teedev,
1219	struct tee_ioctl_version_data *vers)
1220	{
1221	struct tee_ioctl_version_data v = {
1222	.impl_id = TEE_IMPL_ID_OPTEE,
1223	.impl_caps = TEE_OPTEE_CAP_TZ,
1224	.gen_caps = TEE_GEN_CAP_GP,
1225	};
1226	struct optee *optee = tee_get_drvdata(teedev);
1227
1228	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1229	v.gen_caps |= TEE_GEN_CAP_REG_MEM;
1230	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
1231	v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
1232	*vers = v;
1233	}
1234
1235	static int optee_smc_open(struct tee_context *ctx)
1236	{
1237	struct optee *optee = tee_get_drvdata(teedev: ctx->teedev);
1238	u32 sec_caps = optee->smc.sec_caps;
1239
1240	return optee_open(ctx, cap_memref_null: sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
1241	}
1242
1243	static const struct tee_driver_ops optee_clnt_ops = {
1244	.get_version = optee_get_version,
1245	.open = optee_smc_open,
1246	.release = optee_release,
1247	.open_session = optee_open_session,
1248	.close_session = optee_close_session,
1249	.system_session = optee_system_session,
1250	.invoke_func = optee_invoke_func,
1251	.cancel_req = optee_cancel_req,
1252	.shm_register = optee_shm_register,
1253	.shm_unregister = optee_shm_unregister,
1254	};
1255
1256	static const struct tee_desc optee_clnt_desc = {
1257	.name = DRIVER_NAME "-clnt",
1258	.ops = &optee_clnt_ops,
1259	.owner = THIS_MODULE,
1260	};
1261
1262	static const struct tee_driver_ops optee_supp_ops = {
1263	.get_version = optee_get_version,
1264	.open = optee_smc_open,
1265	.release = optee_release_supp,
1266	.supp_recv = optee_supp_recv,
1267	.supp_send = optee_supp_send,
1268	.shm_register = optee_shm_register_supp,
1269	.shm_unregister = optee_shm_unregister_supp,
1270	};
1271
1272	static const struct tee_desc optee_supp_desc = {
1273	.name = DRIVER_NAME "-supp",
1274	.ops = &optee_supp_ops,
1275	.owner = THIS_MODULE,
1276	.flags = TEE_DESC_PRIVILEGED,
1277	};
1278
1279	static const struct optee_ops optee_ops = {
1280	.do_call_with_arg = optee_smc_do_call_with_arg,
1281	.to_msg_param = optee_to_msg_param,
1282	.from_msg_param = optee_from_msg_param,
1283	.lend_protmem = optee_smc_lend_protmem,
1284	.reclaim_protmem = optee_smc_reclaim_protmem,
1285	};
1286
1287	static int enable_async_notif(optee_invoke_fn *invoke_fn)
1288	{
1289	struct arm_smccc_res res;
1290
1291	invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
1292
1293	if (res.a0)
1294	return -EINVAL;
1295	return 0;
1296	}
1297
1298	static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
1299	{
1300	struct arm_smccc_res res;
1301
1302	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1303
1304	if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
1305	res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
1306	return true;
1307	return false;
1308	}
1309
1310	#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1311	static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
1312	{
1313	struct arm_smccc_res res;
1314
1315	invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1316
1317	if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
1318	res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
1319	res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
1320	res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
1321	return true;
1322	return false;
1323	}
1324	#endif
1325
1326	static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
1327	{
1328	union {
1329	struct arm_smccc_res smccc;
1330	struct optee_smc_call_get_os_revision_result result;
1331	} res = {
1332	.result = {
1333	.build_id = 0
1334	}
1335	};
1336
1337	invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
1338	&res.smccc);
1339
1340	if (res.result.build_id)
1341	pr_info("revision %lu.%lu (%0*lx)", res.result.major,
1342	res.result.minor, (int)sizeof(res.result.build_id) * 2,
1343	res.result.build_id);
1344	else
1345	pr_info("revision %lu.%lu", res.result.major, res.result.minor);
1346	}
1347
1348	static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
1349	{
1350	union {
1351	struct arm_smccc_res smccc;
1352	struct optee_smc_calls_revision_result result;
1353	} res;
1354
1355	invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1356
1357	if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
1358	(int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
1359	return true;
1360	return false;
1361	}
1362
1363	static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
1364	u32 *sec_caps, u32 *max_notif_value,
1365	unsigned int *rpc_param_count)
1366	{
1367	union {
1368	struct arm_smccc_res smccc;
1369	struct optee_smc_exchange_capabilities_result result;
1370	} res;
1371	u32 a1 = 0;
1372
1373	/*
1374	* TODO This isn't enough to tell if it's UP system (from kernel
1375	* point of view) or not, is_smp() returns the information
1376	* needed, but can't be called directly from here.
1377	*/
1378	if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
1379	a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
1380
1381	invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
1382	&res.smccc);
1383
1384	if (res.result.status != OPTEE_SMC_RETURN_OK)
1385	return false;
1386
1387	*sec_caps = res.result.capabilities;
1388	if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
1389	*max_notif_value = res.result.max_notif_value;
1390	else
1391	*max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
1392	if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1393	*rpc_param_count = (u8)res.result.data;
1394	else
1395	*rpc_param_count = 0;
1396
1397	return true;
1398	}
1399
1400	static unsigned int optee_msg_get_thread_count(optee_invoke_fn *invoke_fn)
1401	{
1402	struct arm_smccc_res res;
1403
1404	invoke_fn(OPTEE_SMC_GET_THREAD_COUNT, 0, 0, 0, 0, 0, 0, 0, &res);
1405	if (res.a0)
1406	return 0;
1407	return res.a1;
1408	}
1409
1410	static struct tee_shm_pool *
1411	optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
1412	{
1413	union {
1414	struct arm_smccc_res smccc;
1415	struct optee_smc_get_shm_config_result result;
1416	} res;
1417	unsigned long vaddr;
1418	phys_addr_t paddr;
1419	size_t size;
1420	phys_addr_t begin;
1421	phys_addr_t end;
1422	void *va;
1423	void *rc;
1424
1425	invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1426	if (res.result.status != OPTEE_SMC_RETURN_OK) {
1427	pr_err("static shm service not available\n");
1428	return ERR_PTR(error: -ENOENT);
1429	}
1430
1431	if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
1432	pr_err("only normal cached shared memory supported\n");
1433	return ERR_PTR(error: -EINVAL);
1434	}
1435
1436	begin = roundup(res.result.start, PAGE_SIZE);
1437	end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
1438	paddr = begin;
1439	size = end - begin;
1440
1441	va = memremap(offset: paddr, size, flags: MEMREMAP_WB);
1442	if (!va) {
1443	pr_err("shared memory ioremap failed\n");
1444	return ERR_PTR(error: -EINVAL);
1445	}
1446	vaddr = (unsigned long)va;
1447
1448	rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
1449	OPTEE_MIN_STATIC_POOL_ALIGN);
1450	if (IS_ERR(ptr: rc))
1451	memunmap(addr: va);
1452	else
1453	*memremaped_shm = va;
1454
1455	return rc;
1456	}
1457
1458	/* Simple wrapper functions to be able to use a function pointer */
1459	static void optee_smccc_smc(unsigned long a0, unsigned long a1,
1460	unsigned long a2, unsigned long a3,
1461	unsigned long a4, unsigned long a5,
1462	unsigned long a6, unsigned long a7,
1463	struct arm_smccc_res *res)
1464	{
1465	arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1466	}
1467
1468	static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
1469	unsigned long a2, unsigned long a3,
1470	unsigned long a4, unsigned long a5,
1471	unsigned long a6, unsigned long a7,
1472	struct arm_smccc_res *res)
1473	{
1474	arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1475	}
1476
1477	static optee_invoke_fn *get_invoke_func(struct device *dev)
1478	{
1479	const char *method;
1480
1481	pr_info("probing for conduit method.\n");
1482
1483	if (device_property_read_string(dev, propname: "method", val: &method)) {
1484	pr_warn("missing \"method\" property\n");
1485	return ERR_PTR(error: -ENXIO);
1486	}
1487
1488	if (!strcmp("hvc", method))
1489	return optee_smccc_hvc;
1490	else if (!strcmp("smc", method))
1491	return optee_smccc_smc;
1492
1493	pr_warn("invalid \"method\" property: %s\n", method);
1494	return ERR_PTR(error: -EINVAL);
1495	}
1496
1497	/* optee_remove - Device Removal Routine
1498	* @pdev: platform device information struct
1499	*
1500	* optee_remove is called by platform subsystem to alert the driver
1501	* that it should release the device
1502	*/
1503	static void optee_smc_remove(struct platform_device *pdev)
1504	{
1505	struct optee *optee = platform_get_drvdata(pdev);
1506
1507	/*
1508	* Ask OP-TEE to free all cached shared memory objects to decrease
1509	* reference counters and also avoid wild pointers in secure world
1510	* into the old shared memory range.
1511	*/
1512	if (!optee->rpc_param_count)
1513	optee_disable_shm_cache(optee);
1514
1515	optee_smc_notif_uninit_irq(optee);
1516
1517	optee_remove_common(optee);
1518
1519	if (optee->smc.memremaped_shm)
1520	memunmap(addr: optee->smc.memremaped_shm);
1521
1522	kfree(objp: optee);
1523	}
1524
1525	/* optee_shutdown - Device Removal Routine
1526	* @pdev: platform device information struct
1527	*
1528	* platform_shutdown is called by the platform subsystem to alert
1529	* the driver that a shutdown, reboot, or kexec is happening and
1530	* device must be disabled.
1531	*/
1532	static void optee_shutdown(struct platform_device *pdev)
1533	{
1534	struct optee *optee = platform_get_drvdata(pdev);
1535
1536	if (!optee->rpc_param_count)
1537	optee_disable_shm_cache(optee);
1538	}
1539
1540	#ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1541
1542	#define OPTEE_FW_IMAGE "optee/tee.bin"
1543
1544	static optee_invoke_fn *cpuhp_invoke_fn;
1545
1546	static int optee_cpuhp_probe(unsigned int cpu)
1547	{
1548	/*
1549	* Invoking a call on a CPU will cause OP-TEE to perform the required
1550	* setup for that CPU. Just invoke the call to get the UID since that
1551	* has no side effects.
1552	*/
1553	if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
1554	return 0;
1555	else
1556	return -EINVAL;
1557	}
1558
1559	static int optee_load_fw(struct platform_device *pdev,
1560	optee_invoke_fn *invoke_fn)
1561	{
1562	const struct firmware *fw = NULL;
1563	struct arm_smccc_res res;
1564	phys_addr_t data_pa;
1565	u8 *data_buf = NULL;
1566	u64 data_size;
1567	u32 data_pa_high, data_pa_low;
1568	u32 data_size_high, data_size_low;
1569	int rc;
1570	int hp_state;
1571
1572	if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
1573	return 0;
1574
1575	rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
1576	if (rc) {
1577	/*
1578	* The firmware in the rootfs will not be accessible until we
1579	* are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
1580	* that point.
1581	*/
1582	if (system_state < SYSTEM_RUNNING)
1583	return -EPROBE_DEFER;
1584	goto fw_err;
1585	}
1586
1587	data_size = fw->size;
1588	/*
1589	* This uses the GFP_DMA flag to ensure we are allocated memory in the
1590	* 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
1591	*/
1592	data_buf = kmemdup(fw->data, fw->size, GFP_KERNEL | GFP_DMA);
1593	if (!data_buf) {
1594	rc = -ENOMEM;
1595	goto fw_err;
1596	}
1597	data_pa = virt_to_phys(data_buf);
1598	reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
1599	reg_pair_from_64(&data_size_high, &data_size_low, data_size);
1600	goto fw_load;
1601
1602	fw_err:
1603	pr_warn("image loading failed\n");
1604	data_pa_high = 0;
1605	data_pa_low = 0;
1606	data_size_high = 0;
1607	data_size_low = 0;
1608
1609	fw_load:
1610	/*
1611	* Always invoke the SMC, even if loading the image fails, to indicate
1612	* to EL3 that we have passed the point where it should allow invoking
1613	* this SMC.
1614	*/
1615	pr_warn("OP-TEE image loaded from kernel, this can be insecure");
1616	invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
1617	data_pa_high, data_pa_low, 0, 0, 0, &res);
1618	if (!rc)
1619	rc = res.a0;
1620	release_firmware(fw);
1621	kfree(data_buf);
1622
1623	if (!rc) {
1624	/*
1625	* We need to initialize OP-TEE on all other running cores as
1626	* well. Any cores that aren't running yet will get initialized
1627	* when they are brought up by the power management functions in
1628	* TF-A which are registered by the OP-TEE SPD. Due to that we
1629	* can un-register the callback right after registering it.
1630	*/
1631	cpuhp_invoke_fn = invoke_fn;
1632	hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
1633	optee_cpuhp_probe, NULL);
1634	if (hp_state < 0) {
1635	pr_warn("Failed with CPU hotplug setup for OP-TEE");
1636	return -EINVAL;
1637	}
1638	cpuhp_remove_state(hp_state);
1639	cpuhp_invoke_fn = NULL;
1640	}
1641
1642	return rc;
1643	}
1644	#else
1645	static inline int optee_load_fw(struct platform_device *pdev,
1646	optee_invoke_fn *invoke_fn)
1647	{
1648	return 0;
1649	}
1650	#endif
1651
1652	static struct tee_protmem_pool *static_protmem_pool_init(struct optee *optee)
1653	{
1654	#if IS_ENABLED(CONFIG_OPTEE_STATIC_PROTMEM_POOL)
1655	union {
1656	struct arm_smccc_res smccc;
1657	struct optee_smc_get_protmem_config_result result;
1658	} res;
1659	struct tee_protmem_pool *pool;
1660	void *p;
1661	int rc;
1662
1663	optee->smc.invoke_fn(OPTEE_SMC_GET_PROTMEM_CONFIG, 0, 0, 0, 0,
1664	0, 0, 0, &res.smccc);
1665	if (res.result.status != OPTEE_SMC_RETURN_OK)
1666	return ERR_PTR(error: -EINVAL);
1667
1668	rc = optee_set_dma_mask(optee, pa_width: res.result.pa_width);
1669	if (rc)
1670	return ERR_PTR(error: rc);
1671
1672	/*
1673	* Map the memory as uncached to make sure the kernel can work with
1674	* __pfn_to_page() and friends since that's needed when passing the
1675	* protected DMA-buf to a device. The memory should otherwise not
1676	* be touched by the kernel since it's likely to cause an external
1677	* abort due to the protection status.
1678	*/
1679	p = devm_memremap(dev: &optee->teedev->dev, offset: res.result.start,
1680	size: res.result.size, flags: MEMREMAP_WC);
1681	if (IS_ERR(ptr: p))
1682	return p;
1683
1684	pool = tee_protmem_static_pool_alloc(paddr: res.result.start, size: res.result.size);
1685	if (IS_ERR(ptr: pool))
1686	devm_memunmap(dev: &optee->teedev->dev, addr: p);
1687
1688	return pool;
1689	#else
1690	return ERR_PTR(-EINVAL);
1691	#endif
1692	}
1693
1694	static int optee_protmem_pool_init(struct optee *optee)
1695	{
1696	bool protm = optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_PROTMEM;
1697	bool dyn_protm = optee->smc.sec_caps &
1698	OPTEE_SMC_SEC_CAP_DYNAMIC_PROTMEM;
1699	enum tee_dma_heap_id heap_id = TEE_DMA_HEAP_SECURE_VIDEO_PLAY;
1700	struct tee_protmem_pool *pool = ERR_PTR(error: -EINVAL);
1701	int rc = -EINVAL;
1702
1703	if (!protm && !dyn_protm)
1704	return 0;
1705
1706	if (protm)
1707	pool = static_protmem_pool_init(optee);
1708	if (dyn_protm && IS_ERR(ptr: pool))
1709	pool = optee_protmem_alloc_dyn_pool(optee, id: heap_id);
1710	if (IS_ERR(ptr: pool))
1711	return PTR_ERR(ptr: pool);
1712
1713	rc = tee_device_register_dma_heap(teedev: optee->teedev, id: heap_id, pool);
1714	if (rc)
1715	pool->ops->destroy_pool(pool);
1716
1717	return rc;
1718	}
1719
1720	static int optee_probe(struct platform_device *pdev)
1721	{
1722	optee_invoke_fn *invoke_fn;
1723	struct tee_shm_pool *pool = ERR_PTR(error: -EINVAL);
1724	struct optee *optee = NULL;
1725	void *memremaped_shm = NULL;
1726	unsigned int rpc_param_count;
1727	unsigned int thread_count;
1728	struct tee_device *teedev;
1729	struct tee_context *ctx;
1730	u32 max_notif_value;
1731	u32 arg_cache_flags;
1732	u32 sec_caps;
1733	int rc;
1734
1735	invoke_fn = get_invoke_func(dev: &pdev->dev);
1736	if (IS_ERR(ptr: invoke_fn))
1737	return PTR_ERR(ptr: invoke_fn);
1738
1739	rc = optee_load_fw(pdev, invoke_fn);
1740	if (rc)
1741	return rc;
1742
1743	if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
1744	pr_warn("api uid mismatch\n");
1745	return -EINVAL;
1746	}
1747
1748	optee_msg_get_os_revision(invoke_fn);
1749
1750	if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
1751	pr_warn("api revision mismatch\n");
1752	return -EINVAL;
1753	}
1754
1755	thread_count = optee_msg_get_thread_count(invoke_fn);
1756	if (!optee_msg_exchange_capabilities(invoke_fn, sec_caps: &sec_caps,
1757	max_notif_value: &max_notif_value,
1758	rpc_param_count: &rpc_param_count)) {
1759	pr_warn("capabilities mismatch\n");
1760	return -EINVAL;
1761	}
1762
1763	/*
1764	* Try to use dynamic shared memory if possible
1765	*/
1766	if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
1767	/*
1768	* If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
1769	* optee_get_msg_arg() to pre-register (by having
1770	* OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
1771	* an argument struct.
1772	*
1773	* With the page is pre-registered we can use a non-zero
1774	* offset for argument struct, this is indicated with
1775	* OPTEE_SHM_ARG_SHARED.
1776	*
1777	* This means that optee_smc_do_call_with_arg() will use
1778	* OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
1779	*/
1780	if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1781	arg_cache_flags = OPTEE_SHM_ARG_SHARED;
1782	else
1783	arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
1784
1785	pool = optee_shm_pool_alloc_pages();
1786	}
1787
1788	/*
1789	* If dynamic shared memory is not available or failed - try static one
1790	*/
1791	if (IS_ERR(ptr: pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
1792	/*
1793	* The static memory pool can use non-zero page offsets so
1794	* let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
1795	*
1796	* optee_get_msg_arg() should not pre-register the
1797	* allocated page used to pass an argument struct, this is
1798	* indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
1799	*
1800	* This means that optee_smc_do_call_with_arg() will use
1801	* OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
1802	* OPTEE_SMC_CALL_WITH_RPC_ARG.
1803	*/
1804	arg_cache_flags = OPTEE_SHM_ARG_SHARED |
1805	OPTEE_SHM_ARG_ALLOC_PRIV;
1806	pool = optee_config_shm_memremap(invoke_fn, memremaped_shm: &memremaped_shm);
1807	}
1808
1809	if (IS_ERR(ptr: pool))
1810	return PTR_ERR(ptr: pool);
1811
1812	optee = kzalloc(sizeof(*optee), GFP_KERNEL);
1813	if (!optee) {
1814	rc = -ENOMEM;
1815	goto err_free_shm_pool;
1816	}
1817
1818	optee->ops = &optee_ops;
1819	optee->smc.invoke_fn = invoke_fn;
1820	optee->smc.sec_caps = sec_caps;
1821	optee->rpc_param_count = rpc_param_count;
1822
1823	if (IS_REACHABLE(CONFIG_RPMB) &&
1824	(sec_caps & OPTEE_SMC_SEC_CAP_RPMB_PROBE))
1825	optee->in_kernel_rpmb_routing = true;
1826
1827	teedev = tee_device_alloc(teedesc: &optee_clnt_desc, NULL, pool, driver_data: optee);
1828	if (IS_ERR(ptr: teedev)) {
1829	rc = PTR_ERR(ptr: teedev);
1830	goto err_free_optee;
1831	}
1832	optee->teedev = teedev;
1833
1834	teedev = tee_device_alloc(teedesc: &optee_supp_desc, NULL, pool, driver_data: optee);
1835	if (IS_ERR(ptr: teedev)) {
1836	rc = PTR_ERR(ptr: teedev);
1837	goto err_unreg_teedev;
1838	}
1839	optee->supp_teedev = teedev;
1840
1841	optee_set_dev_group(optee);
1842
1843	rc = tee_device_register(teedev: optee->teedev);
1844	if (rc)
1845	goto err_unreg_supp_teedev;
1846
1847	rc = tee_device_register(teedev: optee->supp_teedev);
1848	if (rc)
1849	goto err_unreg_supp_teedev;
1850
1851	optee_cq_init(cq: &optee->call_queue, thread_count);
1852	optee_supp_init(supp: &optee->supp);
1853	optee->smc.memremaped_shm = memremaped_shm;
1854	optee->pool = pool;
1855	optee_shm_arg_cache_init(optee, flags: arg_cache_flags);
1856	mutex_init(&optee->rpmb_dev_mutex);
1857
1858	platform_set_drvdata(pdev, data: optee);
1859	ctx = teedev_open(teedev: optee->teedev);
1860	if (IS_ERR(ptr: ctx)) {
1861	rc = PTR_ERR(ptr: ctx);
1862	goto err_supp_uninit;
1863	}
1864	optee->ctx = ctx;
1865	rc = optee_notif_init(optee, max_key: max_notif_value);
1866	if (rc)
1867	goto err_close_ctx;
1868
1869	if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1870	unsigned int irq;
1871
1872	rc = platform_get_irq(pdev, 0);
1873	if (rc < 0) {
1874	pr_err("platform_get_irq: ret %d\n", rc);
1875	goto err_notif_uninit;
1876	}
1877	irq = rc;
1878
1879	rc = optee_smc_notif_init_irq(optee, irq);
1880	if (rc) {
1881	irq_dispose_mapping(virq: irq);
1882	goto err_notif_uninit;
1883	}
1884	enable_async_notif(invoke_fn: optee->smc.invoke_fn);
1885	pr_info("Asynchronous notifications enabled\n");
1886	}
1887
1888	if (optee_protmem_pool_init(optee))
1889	pr_info("Protected memory service not available\n");
1890
1891	/*
1892	* Ensure that there are no pre-existing shm objects before enabling
1893	* the shm cache so that there's no chance of receiving an invalid
1894	* address during shutdown. This could occur, for example, if we're
1895	* kexec booting from an older kernel that did not properly cleanup the
1896	* shm cache.
1897	*/
1898	optee_disable_unmapped_shm_cache(optee);
1899
1900	/*
1901	* Only enable the shm cache in case we're not able to pass the RPC
1902	* arg struct right after the normal arg struct.
1903	*/
1904	if (!optee->rpc_param_count)
1905	optee_enable_shm_cache(optee);
1906
1907	if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1908	pr_info("dynamic shared memory is enabled\n");
1909
1910	rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1911	if (rc)
1912	goto err_disable_shm_cache;
1913
1914	INIT_WORK(&optee->rpmb_scan_bus_work, optee_bus_scan_rpmb);
1915	optee->rpmb_intf.notifier_call = optee_rpmb_intf_rdev;
1916	blocking_notifier_chain_register(nh: &optee_rpmb_intf_added,
1917	nb: &optee->rpmb_intf);
1918	pr_info("initialized driver\n");
1919	return 0;
1920
1921	err_disable_shm_cache:
1922	if (!optee->rpc_param_count)
1923	optee_disable_shm_cache(optee);
1924	optee_smc_notif_uninit_irq(optee);
1925	optee_unregister_devices();
1926	err_notif_uninit:
1927	optee_notif_uninit(optee);
1928	err_close_ctx:
1929	teedev_close_context(ctx);
1930	err_supp_uninit:
1931	rpmb_dev_put(rdev: optee->rpmb_dev);
1932	mutex_destroy(lock: &optee->rpmb_dev_mutex);
1933	optee_shm_arg_cache_uninit(optee);
1934	optee_supp_uninit(supp: &optee->supp);
1935	mutex_destroy(lock: &optee->call_queue.mutex);
1936	err_unreg_supp_teedev:
1937	tee_device_unregister(teedev: optee->supp_teedev);
1938	err_unreg_teedev:
1939	tee_device_unregister(teedev: optee->teedev);
1940	err_free_optee:
1941	kfree(objp: optee);
1942	err_free_shm_pool:
1943	tee_shm_pool_free(pool);
1944	if (memremaped_shm)
1945	memunmap(addr: memremaped_shm);
1946	return rc;
1947	}
1948
1949	static const struct of_device_id optee_dt_match[] = {
1950	{ .compatible = "linaro,optee-tz" },
1951	{},
1952	};
1953	MODULE_DEVICE_TABLE(of, optee_dt_match);
1954
1955	static struct platform_driver optee_driver = {
1956	.probe = optee_probe,
1957	.remove = optee_smc_remove,
1958	.shutdown = optee_shutdown,
1959	.driver = {
1960	.name = "optee",
1961	.of_match_table = optee_dt_match,
1962	},
1963	};
1964
1965	int optee_smc_abi_register(void)
1966	{
1967	return platform_driver_register(&optee_driver);
1968	}
1969
1970	void optee_smc_abi_unregister(void)
1971	{
1972	platform_driver_unregister(&optee_driver);
1973	}
1974