1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Dynamic DMA mapping support.
4	*
5	* This implementation is a fallback for platforms that do not support
6	* I/O TLBs (aka DMA address translation hardware).
7	* Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
8	* Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
9	* Copyright (C) 2000, 2003 Hewlett-Packard Co
10	* David Mosberger-Tang <davidm@hpl.hp.com>
11	*
12	* 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13	* 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14	* unnecessary i-cache flushing.
15	* 04/07/.. ak Better overflow handling. Assorted fixes.
16	* 05/09/10 linville Add support for syncing ranges, support syncing for
17	* DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18	* 08/12/11 beckyb Add highmem support
19	*/
20
21	#define pr_fmt(fmt) "software IO TLB: " fmt
22
23	#include <linux/cache.h>
24	#include <linux/cc_platform.h>
25	#include <linux/ctype.h>
26	#include <linux/debugfs.h>
27	#include <linux/dma-direct.h>
28	#include <linux/dma-map-ops.h>
29	#include <linux/export.h>
30	#include <linux/gfp.h>
31	#include <linux/highmem.h>
32	#include <linux/io.h>
33	#include <linux/iommu-helper.h>
34	#include <linux/init.h>
35	#include <linux/memblock.h>
36	#include <linux/mm.h>
37	#include <linux/pfn.h>
38	#include <linux/rculist.h>
39	#include <linux/scatterlist.h>
40	#include <linux/set_memory.h>
41	#include <linux/spinlock.h>
42	#include <linux/string.h>
43	#include <linux/swiotlb.h>
44	#include <linux/types.h>
45	#ifdef CONFIG_DMA_RESTRICTED_POOL
46	#include <linux/of.h>
47	#include <linux/of_fdt.h>
48	#include <linux/of_reserved_mem.h>
49	#include <linux/slab.h>
50	#endif
51
52	#define CREATE_TRACE_POINTS
53	#include <trace/events/swiotlb.h>
54
55	#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
56
57	/*
58	* Minimum IO TLB size to bother booting with. Systems with mainly
59	* 64bit capable cards will only lightly use the swiotlb. If we can't
60	* allocate a contiguous 1MB, we're probably in trouble anyway.
61	*/
62	#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
63
64	/**
65	* struct io_tlb_slot - IO TLB slot descriptor
66	* @orig_addr: The original address corresponding to a mapped entry.
67	* @alloc_size: Size of the allocated buffer.
68	* @list: The free list describing the number of free entries available
69	* from each index.
70	* @pad_slots: Number of preceding padding slots. Valid only in the first
71	* allocated non-padding slot.
72	*/
73	struct io_tlb_slot {
74	phys_addr_t orig_addr;
75	size_t alloc_size;
76	unsigned short list;
77	unsigned short pad_slots;
78	};
79
80	static bool swiotlb_force_bounce;
81	static bool swiotlb_force_disable;
82
83	#ifdef CONFIG_SWIOTLB_DYNAMIC
84
85	static void swiotlb_dyn_alloc(struct work_struct *work);
86
87	static struct io_tlb_mem io_tlb_default_mem = {
88	.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
89	.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
90	.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
91	swiotlb_dyn_alloc),
92	};
93
94	#else /* !CONFIG_SWIOTLB_DYNAMIC */
95
96	static struct io_tlb_mem io_tlb_default_mem;
97
98	#endif /* CONFIG_SWIOTLB_DYNAMIC */
99
100	static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
101	static unsigned long default_nareas;
102
103	/**
104	* struct io_tlb_area - IO TLB memory area descriptor
105	*
106	* This is a single area with a single lock.
107	*
108	* @used: The number of used IO TLB block.
109	* @index: The slot index to start searching in this area for next round.
110	* @lock: The lock to protect the above data structures in the map and
111	* unmap calls.
112	*/
113	struct io_tlb_area {
114	unsigned long used;
115	unsigned int index;
116	spinlock_t lock;
117	};
118
119	/*
120	* Round up number of slabs to the next power of 2. The last area is going
121	* be smaller than the rest if default_nslabs is not power of two.
122	* The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
123	* otherwise a segment may span two or more areas. It conflicts with free
124	* contiguous slots tracking: free slots are treated contiguous no matter
125	* whether they cross an area boundary.
126	*
127	* Return true if default_nslabs is rounded up.
128	*/
129	static bool round_up_default_nslabs(void)
130	{
131	if (!default_nareas)
132	return false;
133
134	if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
135	default_nslabs = IO_TLB_SEGSIZE * default_nareas;
136	else if (is_power_of_2(n: default_nslabs))
137	return false;
138	default_nslabs = roundup_pow_of_two(default_nslabs);
139	return true;
140	}
141
142	/**
143	* swiotlb_adjust_nareas() - adjust the number of areas and slots
144	* @nareas: Desired number of areas. Zero is treated as 1.
145	*
146	* Adjust the default number of areas in a memory pool.
147	* The default size of the memory pool may also change to meet minimum area
148	* size requirements.
149	*/
150	static void swiotlb_adjust_nareas(unsigned int nareas)
151	{
152	if (!nareas)
153	nareas = 1;
154	else if (!is_power_of_2(n: nareas))
155	nareas = roundup_pow_of_two(nareas);
156
157	default_nareas = nareas;
158
159	pr_info("area num %d.\n", nareas);
160	if (round_up_default_nslabs())
161	pr_info("SWIOTLB bounce buffer size roundup to %luMB",
162	(default_nslabs << IO_TLB_SHIFT) >> 20);
163	}
164
165	/**
166	* limit_nareas() - get the maximum number of areas for a given memory pool size
167	* @nareas: Desired number of areas.
168	* @nslots: Total number of slots in the memory pool.
169	*
170	* Limit the number of areas to the maximum possible number of areas in
171	* a memory pool of the given size.
172	*
173	* Return: Maximum possible number of areas.
174	*/
175	static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
176	{
177	if (nslots < nareas * IO_TLB_SEGSIZE)
178	return nslots / IO_TLB_SEGSIZE;
179	return nareas;
180	}
181
182	static int __init
183	setup_io_tlb_npages(char *str)
184	{
185	if (isdigit(c: *str)) {
186	/* avoid tail segment of size < IO_TLB_SEGSIZE */
187	default_nslabs =
188	ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
189	}
190	if (*str == ',')
191	++str;
192	if (isdigit(c: *str))
193	swiotlb_adjust_nareas(nareas: simple_strtoul(str, &str, 0));
194	if (*str == ',')
195	++str;
196	if (!strcmp(str, "force"))
197	swiotlb_force_bounce = true;
198	else if (!strcmp(str, "noforce"))
199	swiotlb_force_disable = true;
200
201	return 0;
202	}
203	early_param("swiotlb", setup_io_tlb_npages);
204
205	unsigned long swiotlb_size_or_default(void)
206	{
207	return default_nslabs << IO_TLB_SHIFT;
208	}
209
210	void __init swiotlb_adjust_size(unsigned long size)
211	{
212	/*
213	* If swiotlb parameter has not been specified, give a chance to
214	* architectures such as those supporting memory encryption to
215	* adjust/expand SWIOTLB size for their use.
216	*/
217	if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
218	return;
219
220	size = ALIGN(size, IO_TLB_SIZE);
221	default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
222	if (round_up_default_nslabs())
223	size = default_nslabs << IO_TLB_SHIFT;
224	pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
225	}
226
227	void swiotlb_print_info(void)
228	{
229	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
230
231	if (!mem->nslabs) {
232	pr_warn("No low mem\n");
233	return;
234	}
235
236	pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
237	(mem->nslabs << IO_TLB_SHIFT) >> 20);
238	}
239
240	static inline unsigned long io_tlb_offset(unsigned long val)
241	{
242	return val & (IO_TLB_SEGSIZE - 1);
243	}
244
245	static inline unsigned long nr_slots(u64 val)
246	{
247	return DIV_ROUND_UP(val, IO_TLB_SIZE);
248	}
249
250	/*
251	* Early SWIOTLB allocation may be too early to allow an architecture to
252	* perform the desired operations. This function allows the architecture to
253	* call SWIOTLB when the operations are possible. It needs to be called
254	* before the SWIOTLB memory is used.
255	*/
256	void __init swiotlb_update_mem_attributes(void)
257	{
258	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
259	unsigned long bytes;
260
261	if (!mem->nslabs || mem->late_alloc)
262	return;
263	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
264	set_memory_decrypted(addr: (unsigned long)mem->vaddr, numpages: bytes >> PAGE_SHIFT);
265	}
266
267	static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
268	unsigned long nslabs, bool late_alloc, unsigned int nareas)
269	{
270	void *vaddr = phys_to_virt(address: start);
271	unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
272
273	mem->nslabs = nslabs;
274	mem->start = start;
275	mem->end = mem->start + bytes;
276	mem->late_alloc = late_alloc;
277	mem->nareas = nareas;
278	mem->area_nslabs = nslabs / mem->nareas;
279
280	for (i = 0; i < mem->nareas; i++) {
281	spin_lock_init(&mem->areas[i].lock);
282	mem->areas[i].index = 0;
283	mem->areas[i].used = 0;
284	}
285
286	for (i = 0; i < mem->nslabs; i++) {
287	mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
288	mem->nslabs - i);
289	mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
290	mem->slots[i].alloc_size = 0;
291	mem->slots[i].pad_slots = 0;
292	}
293
294	memset(vaddr, 0, bytes);
295	mem->vaddr = vaddr;
296	return;
297	}
298
299	/**
300	* add_mem_pool() - add a memory pool to the allocator
301	* @mem: Software IO TLB allocator.
302	* @pool: Memory pool to be added.
303	*/
304	static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
305	{
306	#ifdef CONFIG_SWIOTLB_DYNAMIC
307	spin_lock(lock: &mem->lock);
308	list_add_rcu(new: &pool->node, head: &mem->pools);
309	mem->nslabs += pool->nslabs;
310	spin_unlock(lock: &mem->lock);
311	#else
312	mem->nslabs = pool->nslabs;
313	#endif
314	}
315
316	static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
317	unsigned int flags,
318	int (*remap)(void *tlb, unsigned long nslabs))
319	{
320	size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
321	void *tlb;
322
323	/*
324	* By default allocate the bounce buffer memory from low memory, but
325	* allow to pick a location everywhere for hypervisors with guest
326	* memory encryption.
327	*/
328	if (flags & SWIOTLB_ANY)
329	tlb = memblock_alloc(size: bytes, PAGE_SIZE);
330	else
331	tlb = memblock_alloc_low(size: bytes, PAGE_SIZE);
332
333	if (!tlb) {
334	pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
335	__func__, bytes);
336	return NULL;
337	}
338
339	if (remap && remap(tlb, nslabs) < 0) {
340	memblock_free(ptr: tlb, PAGE_ALIGN(bytes));
341	pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
342	return NULL;
343	}
344
345	return tlb;
346	}
347
348	/*
349	* Statically reserve bounce buffer space and initialize bounce buffer data
350	* structures for the software IO TLB used to implement the DMA API.
351	*/
352	void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
353	int (*remap)(void *tlb, unsigned long nslabs))
354	{
355	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
356	unsigned long nslabs;
357	unsigned int nareas;
358	size_t alloc_size;
359	void *tlb;
360
361	if (!addressing_limit && !swiotlb_force_bounce)
362	return;
363	if (swiotlb_force_disable)
364	return;
365
366	io_tlb_default_mem.force_bounce =
367	swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
368
369	#ifdef CONFIG_SWIOTLB_DYNAMIC
370	if (!remap)
371	io_tlb_default_mem.can_grow = true;
372	if (flags & SWIOTLB_ANY)
373	io_tlb_default_mem.phys_limit = virt_to_phys(address: high_memory - 1);
374	else
375	io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
376	#endif
377
378	if (!default_nareas)
379	swiotlb_adjust_nareas(nareas: num_possible_cpus());
380
381	nslabs = default_nslabs;
382	nareas = limit_nareas(nareas: default_nareas, nslots: nslabs);
383	while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
384	if (nslabs <= IO_TLB_MIN_SLABS)
385	return;
386	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
387	nareas = limit_nareas(nareas, nslots: nslabs);
388	}
389
390	if (default_nslabs != nslabs) {
391	pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
392	default_nslabs, nslabs);
393	default_nslabs = nslabs;
394	}
395
396	alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
397	mem->slots = memblock_alloc(size: alloc_size, PAGE_SIZE);
398	if (!mem->slots) {
399	pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
400	__func__, alloc_size, PAGE_SIZE);
401	return;
402	}
403
404	mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
405	nareas), SMP_CACHE_BYTES);
406	if (!mem->areas) {
407	pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
408	return;
409	}
410
411	swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, late_alloc: false, nareas);
412	add_mem_pool(mem: &io_tlb_default_mem, pool: mem);
413
414	if (flags & SWIOTLB_VERBOSE)
415	swiotlb_print_info();
416	}
417
418	void __init swiotlb_init(bool addressing_limit, unsigned int flags)
419	{
420	swiotlb_init_remap(addressing_limit, flags, NULL);
421	}
422
423	/*
424	* Systems with larger DMA zones (those that don't support ISA) can
425	* initialize the swiotlb later using the slab allocator if needed.
426	* This should be just like above, but with some error catching.
427	*/
428	int swiotlb_init_late(size_t size, gfp_t gfp_mask,
429	int (*remap)(void *tlb, unsigned long nslabs))
430	{
431	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
432	unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
433	unsigned int nareas;
434	unsigned char *vstart = NULL;
435	unsigned int order, area_order;
436	bool retried = false;
437	int rc = 0;
438
439	if (io_tlb_default_mem.nslabs)
440	return 0;
441
442	if (swiotlb_force_disable)
443	return 0;
444
445	io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
446
447	#ifdef CONFIG_SWIOTLB_DYNAMIC
448	if (!remap)
449	io_tlb_default_mem.can_grow = true;
450	if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
451	io_tlb_default_mem.phys_limit = zone_dma_limit;
452	else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
453	io_tlb_default_mem.phys_limit = max(DMA_BIT_MASK(32), zone_dma_limit);
454	else
455	io_tlb_default_mem.phys_limit = virt_to_phys(address: high_memory - 1);
456	#endif
457
458	if (!default_nareas)
459	swiotlb_adjust_nareas(nareas: num_possible_cpus());
460
461	retry:
462	order = get_order(size: nslabs << IO_TLB_SHIFT);
463	nslabs = SLABS_PER_PAGE << order;
464
465	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
466	vstart = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
467	order);
468	if (vstart)
469	break;
470	order--;
471	nslabs = SLABS_PER_PAGE << order;
472	retried = true;
473	}
474
475	if (!vstart)
476	return -ENOMEM;
477
478	if (remap)
479	rc = remap(vstart, nslabs);
480	if (rc) {
481	free_pages(addr: (unsigned long)vstart, order);
482
483	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
484	if (nslabs < IO_TLB_MIN_SLABS)
485	return rc;
486	retried = true;
487	goto retry;
488	}
489
490	if (retried) {
491	pr_warn("only able to allocate %ld MB\n",
492	(PAGE_SIZE << order) >> 20);
493	}
494
495	nareas = limit_nareas(nareas: default_nareas, nslots: nslabs);
496	area_order = get_order(array_size(sizeof(*mem->areas), nareas));
497	mem->areas = (struct io_tlb_area *)
498	__get_free_pages(GFP_KERNEL | __GFP_ZERO, area_order);
499	if (!mem->areas)
500	goto error_area;
501
502	mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
503	get_order(array_size(sizeof(*mem->slots), nslabs)));
504	if (!mem->slots)
505	goto error_slots;
506
507	set_memory_decrypted(addr: (unsigned long)vstart,
508	numpages: (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
509	swiotlb_init_io_tlb_pool(mem, virt_to_phys(address: vstart), nslabs, late_alloc: true,
510	nareas);
511	add_mem_pool(mem: &io_tlb_default_mem, pool: mem);
512
513	swiotlb_print_info();
514	return 0;
515
516	error_slots:
517	free_pages(addr: (unsigned long)mem->areas, order: area_order);
518	error_area:
519	free_pages(addr: (unsigned long)vstart, order);
520	return -ENOMEM;
521	}
522
523	void __init swiotlb_exit(void)
524	{
525	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
526	unsigned long tbl_vaddr;
527	size_t tbl_size, slots_size;
528	unsigned int area_order;
529
530	if (swiotlb_force_bounce)
531	return;
532
533	if (!mem->nslabs)
534	return;
535
536	pr_info("tearing down default memory pool\n");
537	tbl_vaddr = (unsigned long)phys_to_virt(address: mem->start);
538	tbl_size = PAGE_ALIGN(mem->end - mem->start);
539	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
540
541	set_memory_encrypted(addr: tbl_vaddr, numpages: tbl_size >> PAGE_SHIFT);
542	if (mem->late_alloc) {
543	area_order = get_order(array_size(sizeof(*mem->areas),
544	mem->nareas));
545	free_pages(addr: (unsigned long)mem->areas, order: area_order);
546	free_pages(addr: tbl_vaddr, order: get_order(size: tbl_size));
547	free_pages(addr: (unsigned long)mem->slots, order: get_order(size: slots_size));
548	} else {
549	memblock_free_late(__pa(mem->areas),
550	array_size(sizeof(*mem->areas), mem->nareas));
551	memblock_free_late(base: mem->start, size: tbl_size);
552	memblock_free_late(__pa(mem->slots), size: slots_size);
553	}
554
555	memset(mem, 0, sizeof(*mem));
556	}
557
558	#ifdef CONFIG_SWIOTLB_DYNAMIC
559
560	/**
561	* alloc_dma_pages() - allocate pages to be used for DMA
562	* @gfp: GFP flags for the allocation.
563	* @bytes: Size of the buffer.
564	* @phys_limit: Maximum allowed physical address of the buffer.
565	*
566	* Allocate pages from the buddy allocator. If successful, make the allocated
567	* pages decrypted that they can be used for DMA.
568	*
569	* Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
570	* if the allocated physical address was above @phys_limit.
571	*/
572	static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
573	{
574	unsigned int order = get_order(size: bytes);
575	struct page *page;
576	phys_addr_t paddr;
577	void *vaddr;
578
579	page = alloc_pages(gfp, order);
580	if (!page)
581	return NULL;
582
583	paddr = page_to_phys(page);
584	if (paddr + bytes - 1 > phys_limit) {
585	__free_pages(page, order);
586	return ERR_PTR(error: -EAGAIN);
587	}
588
589	vaddr = phys_to_virt(address: paddr);
590	if (set_memory_decrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
591	goto error;
592	return page;
593
594	error:
595	/* Intentional leak if pages cannot be encrypted again. */
596	if (!set_memory_encrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
597	__free_pages(page, order);
598	return NULL;
599	}
600
601	/**
602	* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
603	* @dev: Device for which a memory pool is allocated.
604	* @bytes: Size of the buffer.
605	* @phys_limit: Maximum allowed physical address of the buffer.
606	* @gfp: GFP flags for the allocation.
607	*
608	* Return: Allocated pages, or %NULL on allocation failure.
609	*/
610	static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
611	u64 phys_limit, gfp_t gfp)
612	{
613	struct page *page;
614
615	/*
616	* Allocate from the atomic pools if memory is encrypted and
617	* the allocation is atomic, because decrypting may block.
618	*/
619	if (!gfpflags_allow_blocking(gfp_flags: gfp) && dev && force_dma_unencrypted(dev)) {
620	void *vaddr;
621
622	if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
623	return NULL;
624
625	return dma_alloc_from_pool(dev, size: bytes, cpu_addr: &vaddr, flags: gfp,
626	phys_addr_ok: dma_coherent_ok);
627	}
628
629	gfp &= ~GFP_ZONEMASK;
630	if (phys_limit <= zone_dma_limit)
631	gfp |= __GFP_DMA;
632	else if (phys_limit <= DMA_BIT_MASK(32))
633	gfp |= __GFP_DMA32;
634
635	while (IS_ERR(ptr: page = alloc_dma_pages(gfp, bytes, phys_limit))) {
636	if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
637	phys_limit < DMA_BIT_MASK(64) &&
638	!(gfp & (__GFP_DMA32 | __GFP_DMA)))
639	gfp |= __GFP_DMA32;
640	else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
641	!(gfp & __GFP_DMA))
642	gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
643	else
644	return NULL;
645	}
646
647	return page;
648	}
649
650	/**
651	* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
652	* @vaddr: Virtual address of the buffer.
653	* @bytes: Size of the buffer.
654	*/
655	static void swiotlb_free_tlb(void *vaddr, size_t bytes)
656	{
657	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
658	dma_free_from_pool(NULL, start: vaddr, size: bytes))
659	return;
660
661	/* Intentional leak if pages cannot be encrypted again. */
662	if (!set_memory_encrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
663	__free_pages(virt_to_page(vaddr), order: get_order(size: bytes));
664	}
665
666	/**
667	* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
668	* @dev: Device for which a memory pool is allocated.
669	* @minslabs: Minimum number of slabs.
670	* @nslabs: Desired (maximum) number of slabs.
671	* @nareas: Number of areas.
672	* @phys_limit: Maximum DMA buffer physical address.
673	* @gfp: GFP flags for the allocations.
674	*
675	* Allocate and initialize a new IO TLB memory pool. The actual number of
676	* slabs may be reduced if allocation of @nslabs fails. If even
677	* @minslabs cannot be allocated, this function fails.
678	*
679	* Return: New memory pool, or %NULL on allocation failure.
680	*/
681	static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
682	unsigned long minslabs, unsigned long nslabs,
683	unsigned int nareas, u64 phys_limit, gfp_t gfp)
684	{
685	struct io_tlb_pool *pool;
686	unsigned int slot_order;
687	struct page *tlb;
688	size_t pool_size;
689	size_t tlb_size;
690
691	if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
692	nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
693	nareas = limit_nareas(nareas, nslots: nslabs);
694	}
695
696	pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
697	pool = kzalloc(pool_size, gfp);
698	if (!pool)
699	goto error;
700	pool->areas = (void *)pool + sizeof(*pool);
701
702	tlb_size = nslabs << IO_TLB_SHIFT;
703	while (!(tlb = swiotlb_alloc_tlb(dev, bytes: tlb_size, phys_limit, gfp))) {
704	if (nslabs <= minslabs)
705	goto error_tlb;
706	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
707	nareas = limit_nareas(nareas, nslots: nslabs);
708	tlb_size = nslabs << IO_TLB_SHIFT;
709	}
710
711	slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
712	pool->slots = (struct io_tlb_slot *)
713	__get_free_pages(gfp, slot_order);
714	if (!pool->slots)
715	goto error_slots;
716
717	swiotlb_init_io_tlb_pool(mem: pool, page_to_phys(tlb), nslabs, late_alloc: true, nareas);
718	return pool;
719
720	error_slots:
721	swiotlb_free_tlb(page_address(tlb), bytes: tlb_size);
722	error_tlb:
723	kfree(objp: pool);
724	error:
725	return NULL;
726	}
727
728	/**
729	* swiotlb_dyn_alloc() - dynamic memory pool allocation worker
730	* @work: Pointer to dyn_alloc in struct io_tlb_mem.
731	*/
732	static void swiotlb_dyn_alloc(struct work_struct *work)
733	{
734	struct io_tlb_mem *mem =
735	container_of(work, struct io_tlb_mem, dyn_alloc);
736	struct io_tlb_pool *pool;
737
738	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, nslabs: default_nslabs,
739	nareas: default_nareas, phys_limit: mem->phys_limit, GFP_KERNEL);
740	if (!pool) {
741	pr_warn_ratelimited("Failed to allocate new pool");
742	return;
743	}
744
745	add_mem_pool(mem, pool);
746	}
747
748	/**
749	* swiotlb_dyn_free() - RCU callback to free a memory pool
750	* @rcu: RCU head in the corresponding struct io_tlb_pool.
751	*/
752	static void swiotlb_dyn_free(struct rcu_head *rcu)
753	{
754	struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
755	size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
756	size_t tlb_size = pool->end - pool->start;
757
758	free_pages(addr: (unsigned long)pool->slots, order: get_order(size: slots_size));
759	swiotlb_free_tlb(vaddr: pool->vaddr, bytes: tlb_size);
760	kfree(objp: pool);
761	}
762
763	/**
764	* __swiotlb_find_pool() - find the IO TLB pool for a physical address
765	* @dev: Device which has mapped the DMA buffer.
766	* @paddr: Physical address within the DMA buffer.
767	*
768	* Find the IO TLB memory pool descriptor which contains the given physical
769	* address, if any. This function is for use only when the dev is known to
770	* be using swiotlb. Use swiotlb_find_pool() for the more general case
771	* when this condition is not met.
772	*
773	* Return: Memory pool which contains @paddr, or %NULL if none.
774	*/
775	struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
776	{
777	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
778	struct io_tlb_pool *pool;
779
780	rcu_read_lock();
781	list_for_each_entry_rcu(pool, &mem->pools, node) {
782	if (paddr >= pool->start && paddr < pool->end)
783	goto out;
784	}
785
786	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
787	if (paddr >= pool->start && paddr < pool->end)
788	goto out;
789	}
790	pool = NULL;
791	out:
792	rcu_read_unlock();
793	return pool;
794	}
795
796	/**
797	* swiotlb_del_pool() - remove an IO TLB pool from a device
798	* @dev: Owning device.
799	* @pool: Memory pool to be removed.
800	*/
801	static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
802	{
803	unsigned long flags;
804
805	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
806	list_del_rcu(entry: &pool->node);
807	spin_unlock_irqrestore(lock: &dev->dma_io_tlb_lock, flags);
808
809	call_rcu(head: &pool->rcu, func: swiotlb_dyn_free);
810	}
811
812	#endif /* CONFIG_SWIOTLB_DYNAMIC */
813
814	/**
815	* swiotlb_dev_init() - initialize swiotlb fields in &struct device
816	* @dev: Device to be initialized.
817	*/
818	void swiotlb_dev_init(struct device *dev)
819	{
820	dev->dma_io_tlb_mem = &io_tlb_default_mem;
821	#ifdef CONFIG_SWIOTLB_DYNAMIC
822	INIT_LIST_HEAD(list: &dev->dma_io_tlb_pools);
823	spin_lock_init(&dev->dma_io_tlb_lock);
824	dev->dma_uses_io_tlb = false;
825	#endif
826	}
827
828	/**
829	* swiotlb_align_offset() - Get required offset into an IO TLB allocation.
830	* @dev: Owning device.
831	* @align_mask: Allocation alignment mask.
832	* @addr: DMA address.
833	*
834	* Return the minimum offset from the start of an IO TLB allocation which is
835	* required for a given buffer address and allocation alignment to keep the
836	* device happy.
837	*
838	* First, the address bits covered by min_align_mask must be identical in the
839	* original address and the bounce buffer address. High bits are preserved by
840	* choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
841	* padding bytes before the bounce buffer.
842	*
843	* Second, @align_mask specifies which bits of the first allocated slot must
844	* be zero. This may require allocating additional padding slots, and then the
845	* offset (in bytes) from the first such padding slot is returned.
846	*/
847	static unsigned int swiotlb_align_offset(struct device *dev,
848	unsigned int align_mask, u64 addr)
849	{
850	return addr & dma_get_min_align_mask(dev) &
851	(align_mask | (IO_TLB_SIZE - 1));
852	}
853
854	/*
855	* Bounce: copy the swiotlb buffer from or back to the original dma location
856	*/
857	static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
858	enum dma_data_direction dir, struct io_tlb_pool *mem)
859	{
860	int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
861	phys_addr_t orig_addr = mem->slots[index].orig_addr;
862	size_t alloc_size = mem->slots[index].alloc_size;
863	unsigned long pfn = PFN_DOWN(orig_addr);
864	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
865	int tlb_offset;
866
867	if (orig_addr == INVALID_PHYS_ADDR)
868	return;
869
870	/*
871	* It's valid for tlb_offset to be negative. This can happen when the
872	* "offset" returned by swiotlb_align_offset() is non-zero, and the
873	* tlb_addr is pointing within the first "offset" bytes of the second
874	* or subsequent slots of the allocated swiotlb area. While it's not
875	* valid for tlb_addr to be pointing within the first "offset" bytes
876	* of the first slot, there's no way to check for such an error since
877	* this function can't distinguish the first slot from the second and
878	* subsequent slots.
879	*/
880	tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
881	swiotlb_align_offset(dev, align_mask: 0, addr: orig_addr);
882
883	orig_addr += tlb_offset;
884	alloc_size -= tlb_offset;
885
886	if (size > alloc_size) {
887	dev_WARN_ONCE(dev, 1,
888	"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
889	alloc_size, size);
890	size = alloc_size;
891	}
892
893	if (PageHighMem(pfn_to_page(pfn))) {
894	unsigned int offset = orig_addr & ~PAGE_MASK;
895	struct page *page;
896	unsigned int sz = 0;
897	unsigned long flags;
898
899	while (size) {
900	sz = min_t(size_t, PAGE_SIZE - offset, size);
901
902	local_irq_save(flags);
903	page = pfn_to_page(pfn);
904	if (dir == DMA_TO_DEVICE)
905	memcpy_from_page(to: vaddr, page, offset, len: sz);
906	else
907	memcpy_to_page(page, offset, from: vaddr, len: sz);
908	local_irq_restore(flags);
909
910	size -= sz;
911	pfn++;
912	vaddr += sz;
913	offset = 0;
914	}
915	} else if (dir == DMA_TO_DEVICE) {
916	memcpy(vaddr, phys_to_virt(orig_addr), size);
917	} else {
918	memcpy(phys_to_virt(orig_addr), vaddr, size);
919	}
920	}
921
922	static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
923	{
924	return start + (idx << IO_TLB_SHIFT);
925	}
926
927	/*
928	* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
929	*/
930	static inline unsigned long get_max_slots(unsigned long boundary_mask)
931	{
932	return (boundary_mask >> IO_TLB_SHIFT) + 1;
933	}
934
935	static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
936	{
937	if (index >= mem->area_nslabs)
938	return 0;
939	return index;
940	}
941
942	/*
943	* Track the total used slots with a global atomic value in order to have
944	* correct information to determine the high water mark. The mem_used()
945	* function gives imprecise results because there's no locking across
946	* multiple areas.
947	*/
948	#ifdef CONFIG_DEBUG_FS
949	static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
950	{
951	unsigned long old_hiwater, new_used;
952
953	new_used = atomic_long_add_return(i: nslots, v: &mem->total_used);
954	old_hiwater = atomic_long_read(v: &mem->used_hiwater);
955	do {
956	if (new_used <= old_hiwater)
957	break;
958	} while (!atomic_long_try_cmpxchg(v: &mem->used_hiwater,
959	old: &old_hiwater, new: new_used));
960	}
961
962	static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
963	{
964	atomic_long_sub(i: nslots, v: &mem->total_used);
965	}
966
967	#else /* !CONFIG_DEBUG_FS */
968	static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
969	{
970	}
971	static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
972	{
973	}
974	#endif /* CONFIG_DEBUG_FS */
975
976	#ifdef CONFIG_SWIOTLB_DYNAMIC
977	#ifdef CONFIG_DEBUG_FS
978	static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
979	{
980	atomic_long_add(i: nslots, v: &mem->transient_nslabs);
981	}
982
983	static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
984	{
985	atomic_long_sub(i: nslots, v: &mem->transient_nslabs);
986	}
987
988	#else /* !CONFIG_DEBUG_FS */
989	static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
990	{
991	}
992	static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
993	{
994	}
995	#endif /* CONFIG_DEBUG_FS */
996	#endif /* CONFIG_SWIOTLB_DYNAMIC */
997
998	/**
999	* swiotlb_search_pool_area() - search one memory area in one pool
1000	* @dev: Device which maps the buffer.
1001	* @pool: Memory pool to be searched.
1002	* @area_index: Index of the IO TLB memory area to be searched.
1003	* @orig_addr: Original (non-bounced) IO buffer address.
1004	* @alloc_size: Total requested size of the bounce buffer,
1005	* including initial alignment padding.
1006	* @alloc_align_mask: Required alignment of the allocated buffer.
1007	*
1008	* Find a suitable sequence of IO TLB entries for the request and allocate
1009	* a buffer from the given IO TLB memory area.
1010	* This function takes care of locking.
1011	*
1012	* Return: Index of the first allocated slot, or -1 on error.
1013	*/
1014	static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1015	int area_index, phys_addr_t orig_addr, size_t alloc_size,
1016	unsigned int alloc_align_mask)
1017	{
1018	struct io_tlb_area *area = pool->areas + area_index;
1019	unsigned long boundary_mask = dma_get_seg_boundary(dev);
1020	dma_addr_t tbl_dma_addr =
1021	phys_to_dma_unencrypted(dev, paddr: pool->start) & boundary_mask;
1022	unsigned long max_slots = get_max_slots(boundary_mask);
1023	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1024	unsigned int nslots = nr_slots(val: alloc_size), stride;
1025	unsigned int offset = swiotlb_align_offset(dev, align_mask: 0, addr: orig_addr);
1026	unsigned int index, slots_checked, count = 0, i;
1027	unsigned long flags;
1028	unsigned int slot_base;
1029	unsigned int slot_index;
1030
1031	BUG_ON(!nslots);
1032	BUG_ON(area_index >= pool->nareas);
1033
1034	/*
1035	* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1036	* page-aligned in the absence of any other alignment requirements.
1037	* 'alloc_align_mask' was later introduced to specify the alignment
1038	* explicitly, however this is passed as zero for streaming mappings
1039	* and so we preserve the old behaviour there in case any drivers are
1040	* relying on it.
1041	*/
1042	if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1043	alloc_align_mask = PAGE_SIZE - 1;
1044
1045	/*
1046	* Ensure that the allocation is at least slot-aligned and update
1047	* 'iotlb_align_mask' to ignore bits that will be preserved when
1048	* offsetting into the allocation.
1049	*/
1050	alloc_align_mask |= (IO_TLB_SIZE - 1);
1051	iotlb_align_mask &= ~alloc_align_mask;
1052
1053	/*
1054	* For mappings with an alignment requirement don't bother looping to
1055	* unaligned slots once we found an aligned one.
1056	*/
1057	stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1058
1059	spin_lock_irqsave(&area->lock, flags);
1060	if (unlikely(nslots > pool->area_nslabs - area->used))
1061	goto not_found;
1062
1063	slot_base = area_index * pool->area_nslabs;
1064	index = area->index;
1065
1066	for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1067	phys_addr_t tlb_addr;
1068
1069	slot_index = slot_base + index;
1070	tlb_addr = slot_addr(start: tbl_dma_addr, idx: slot_index);
1071
1072	if ((tlb_addr & alloc_align_mask) ||
1073	(orig_addr && (tlb_addr & iotlb_align_mask) !=
1074	(orig_addr & iotlb_align_mask))) {
1075	index = wrap_area_index(mem: pool, index: index + 1);
1076	slots_checked++;
1077	continue;
1078	}
1079
1080	if (!iommu_is_span_boundary(index: slot_index, nr: nslots,
1081	shift: nr_slots(val: tbl_dma_addr),
1082	boundary_size: max_slots)) {
1083	if (pool->slots[slot_index].list >= nslots)
1084	goto found;
1085	}
1086	index = wrap_area_index(mem: pool, index: index + stride);
1087	slots_checked += stride;
1088	}
1089
1090	not_found:
1091	spin_unlock_irqrestore(lock: &area->lock, flags);
1092	return -1;
1093
1094	found:
1095	/*
1096	* If we find a slot that indicates we have 'nslots' number of
1097	* contiguous buffers, we allocate the buffers from that slot onwards
1098	* and set the list of free entries to '0' indicating unavailable.
1099	*/
1100	for (i = slot_index; i < slot_index + nslots; i++) {
1101	pool->slots[i].list = 0;
1102	pool->slots[i].alloc_size = alloc_size - (offset +
1103	((i - slot_index) << IO_TLB_SHIFT));
1104	}
1105	for (i = slot_index - 1;
1106	io_tlb_offset(val: i) != IO_TLB_SEGSIZE - 1 &&
1107	pool->slots[i].list; i--)
1108	pool->slots[i].list = ++count;
1109
1110	/*
1111	* Update the indices to avoid searching in the next round.
1112	*/
1113	area->index = wrap_area_index(mem: pool, index: index + nslots);
1114	area->used += nslots;
1115	spin_unlock_irqrestore(lock: &area->lock, flags);
1116
1117	inc_used_and_hiwater(mem: dev->dma_io_tlb_mem, nslots);
1118	return slot_index;
1119	}
1120
1121	#ifdef CONFIG_SWIOTLB_DYNAMIC
1122
1123	/**
1124	* swiotlb_search_area() - search one memory area in all pools
1125	* @dev: Device which maps the buffer.
1126	* @start_cpu: Start CPU number.
1127	* @cpu_offset: Offset from @start_cpu.
1128	* @orig_addr: Original (non-bounced) IO buffer address.
1129	* @alloc_size: Total requested size of the bounce buffer,
1130	* including initial alignment padding.
1131	* @alloc_align_mask: Required alignment of the allocated buffer.
1132	* @retpool: Used memory pool, updated on return.
1133	*
1134	* Search one memory area in all pools for a sequence of slots that match the
1135	* allocation constraints.
1136	*
1137	* Return: Index of the first allocated slot, or -1 on error.
1138	*/
1139	static int swiotlb_search_area(struct device *dev, int start_cpu,
1140	int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1141	unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1142	{
1143	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1144	struct io_tlb_pool *pool;
1145	int area_index;
1146	int index = -1;
1147
1148	rcu_read_lock();
1149	list_for_each_entry_rcu(pool, &mem->pools, node) {
1150	if (cpu_offset >= pool->nareas)
1151	continue;
1152	area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1153	index = swiotlb_search_pool_area(dev, pool, area_index,
1154	orig_addr, alloc_size,
1155	alloc_align_mask);
1156	if (index >= 0) {
1157	*retpool = pool;
1158	break;
1159	}
1160	}
1161	rcu_read_unlock();
1162	return index;
1163	}
1164
1165	/**
1166	* swiotlb_find_slots() - search for slots in the whole swiotlb
1167	* @dev: Device which maps the buffer.
1168	* @orig_addr: Original (non-bounced) IO buffer address.
1169	* @alloc_size: Total requested size of the bounce buffer,
1170	* including initial alignment padding.
1171	* @alloc_align_mask: Required alignment of the allocated buffer.
1172	* @retpool: Used memory pool, updated on return.
1173	*
1174	* Search through the whole software IO TLB to find a sequence of slots that
1175	* match the allocation constraints.
1176	*
1177	* Return: Index of the first allocated slot, or -1 on error.
1178	*/
1179	static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1180	size_t alloc_size, unsigned int alloc_align_mask,
1181	struct io_tlb_pool **retpool)
1182	{
1183	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1184	struct io_tlb_pool *pool;
1185	unsigned long nslabs;
1186	unsigned long flags;
1187	u64 phys_limit;
1188	int cpu, i;
1189	int index;
1190
1191	if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1192	return -1;
1193
1194	cpu = raw_smp_processor_id();
1195	for (i = 0; i < default_nareas; ++i) {
1196	index = swiotlb_search_area(dev, start_cpu: cpu, cpu_offset: i, orig_addr, alloc_size,
1197	alloc_align_mask, retpool: &pool);
1198	if (index >= 0)
1199	goto found;
1200	}
1201
1202	if (!mem->can_grow)
1203	return -1;
1204
1205	schedule_work(work: &mem->dyn_alloc);
1206
1207	nslabs = nr_slots(val: alloc_size);
1208	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1209	pool = swiotlb_alloc_pool(dev, minslabs: nslabs, nslabs, nareas: 1, phys_limit,
1210	GFP_NOWAIT);
1211	if (!pool)
1212	return -1;
1213
1214	index = swiotlb_search_pool_area(dev, pool, area_index: 0, orig_addr,
1215	alloc_size, alloc_align_mask);
1216	if (index < 0) {
1217	swiotlb_dyn_free(rcu: &pool->rcu);
1218	return -1;
1219	}
1220
1221	pool->transient = true;
1222	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1223	list_add_rcu(new: &pool->node, head: &dev->dma_io_tlb_pools);
1224	spin_unlock_irqrestore(lock: &dev->dma_io_tlb_lock, flags);
1225	inc_transient_used(mem, nslots: pool->nslabs);
1226
1227	found:
1228	WRITE_ONCE(dev->dma_uses_io_tlb, true);
1229
1230	/*
1231	* The general barrier orders reads and writes against a presumed store
1232	* of the SWIOTLB buffer address by a device driver (to a driver private
1233	* data structure). It serves two purposes.
1234	*
1235	* First, the store to dev->dma_uses_io_tlb must be ordered before the
1236	* presumed store. This guarantees that the returned buffer address
1237	* cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1238	*
1239	* Second, the load from mem->pools must be ordered before the same
1240	* presumed store. This guarantees that the returned buffer address
1241	* cannot be observed by another CPU before an update of the RCU list
1242	* that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1243	* atomicity).
1244	*
1245	* See also the comment in swiotlb_find_pool().
1246	*/
1247	smp_mb();
1248
1249	*retpool = pool;
1250	return index;
1251	}
1252
1253	#else /* !CONFIG_SWIOTLB_DYNAMIC */
1254
1255	static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1256	size_t alloc_size, unsigned int alloc_align_mask,
1257	struct io_tlb_pool **retpool)
1258	{
1259	struct io_tlb_pool *pool;
1260	int start, i;
1261	int index;
1262
1263	*retpool = pool = &dev->dma_io_tlb_mem->defpool;
1264	i = start = raw_smp_processor_id() & (pool->nareas - 1);
1265	do {
1266	index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1267	alloc_size, alloc_align_mask);
1268	if (index >= 0)
1269	return index;
1270	if (++i >= pool->nareas)
1271	i = 0;
1272	} while (i != start);
1273	return -1;
1274	}
1275
1276	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1277
1278	#ifdef CONFIG_DEBUG_FS
1279
1280	/**
1281	* mem_used() - get number of used slots in an allocator
1282	* @mem: Software IO TLB allocator.
1283	*
1284	* The result is accurate in this version of the function, because an atomic
1285	* counter is available if CONFIG_DEBUG_FS is set.
1286	*
1287	* Return: Number of used slots.
1288	*/
1289	static unsigned long mem_used(struct io_tlb_mem *mem)
1290	{
1291	return atomic_long_read(v: &mem->total_used);
1292	}
1293
1294	#else /* !CONFIG_DEBUG_FS */
1295
1296	/**
1297	* mem_pool_used() - get number of used slots in a memory pool
1298	* @pool: Software IO TLB memory pool.
1299	*
1300	* The result is not accurate, see mem_used().
1301	*
1302	* Return: Approximate number of used slots.
1303	*/
1304	static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1305	{
1306	int i;
1307	unsigned long used = 0;
1308
1309	for (i = 0; i < pool->nareas; i++)
1310	used += pool->areas[i].used;
1311	return used;
1312	}
1313
1314	/**
1315	* mem_used() - get number of used slots in an allocator
1316	* @mem: Software IO TLB allocator.
1317	*
1318	* The result is not accurate, because there is no locking of individual
1319	* areas.
1320	*
1321	* Return: Approximate number of used slots.
1322	*/
1323	static unsigned long mem_used(struct io_tlb_mem *mem)
1324	{
1325	#ifdef CONFIG_SWIOTLB_DYNAMIC
1326	struct io_tlb_pool *pool;
1327	unsigned long used = 0;
1328
1329	rcu_read_lock();
1330	list_for_each_entry_rcu(pool, &mem->pools, node)
1331	used += mem_pool_used(pool);
1332	rcu_read_unlock();
1333
1334	return used;
1335	#else
1336	return mem_pool_used(&mem->defpool);
1337	#endif
1338	}
1339
1340	#endif /* CONFIG_DEBUG_FS */
1341
1342	/**
1343	* swiotlb_tbl_map_single() - bounce buffer map a single contiguous physical area
1344	* @dev: Device which maps the buffer.
1345	* @orig_addr: Original (non-bounced) physical IO buffer address
1346	* @mapping_size: Requested size of the actual bounce buffer, excluding
1347	* any pre- or post-padding for alignment
1348	* @alloc_align_mask: Required start and end alignment of the allocated buffer
1349	* @dir: DMA direction
1350	* @attrs: Optional DMA attributes for the map operation
1351	*
1352	* Find and allocate a suitable sequence of IO TLB slots for the request.
1353	* The allocated space starts at an alignment specified by alloc_align_mask,
1354	* and the size of the allocated space is rounded up so that the total amount
1355	* of allocated space is a multiple of (alloc_align_mask + 1). If
1356	* alloc_align_mask is zero, the allocated space may be at any alignment and
1357	* the size is not rounded up.
1358	*
1359	* The returned address is within the allocated space and matches the bits
1360	* of orig_addr that are specified in the DMA min_align_mask for the device. As
1361	* such, this returned address may be offset from the beginning of the allocated
1362	* space. The bounce buffer space starting at the returned address for
1363	* mapping_size bytes is initialized to the contents of the original IO buffer
1364	* area. Any pre-padding (due to an offset) and any post-padding (due to
1365	* rounding-up the size) is not initialized.
1366	*/
1367	phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1368	size_t mapping_size, unsigned int alloc_align_mask,
1369	enum dma_data_direction dir, unsigned long attrs)
1370	{
1371	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1372	unsigned int offset;
1373	struct io_tlb_pool *pool;
1374	unsigned int i;
1375	size_t size;
1376	int index;
1377	phys_addr_t tlb_addr;
1378	unsigned short pad_slots;
1379
1380	if (!mem || !mem->nslabs) {
1381	dev_warn_ratelimited(dev,
1382	"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1383	return (phys_addr_t)DMA_MAPPING_ERROR;
1384	}
1385
1386	if (cc_platform_has(attr: CC_ATTR_MEM_ENCRYPT))
1387	pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1388
1389	/*
1390	* The default swiotlb memory pool is allocated with PAGE_SIZE
1391	* alignment. If a mapping is requested with larger alignment,
1392	* the mapping may be unable to use the initial slot(s) in all
1393	* sets of IO_TLB_SEGSIZE slots. In such case, a mapping request
1394	* of or near the maximum mapping size would always fail.
1395	*/
1396	dev_WARN_ONCE(dev, alloc_align_mask > ~PAGE_MASK,
1397	"Alloc alignment may prevent fulfilling requests with max mapping_size\n");
1398
1399	offset = swiotlb_align_offset(dev, align_mask: alloc_align_mask, addr: orig_addr);
1400	size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
1401	index = swiotlb_find_slots(dev, orig_addr, alloc_size: size, alloc_align_mask, retpool: &pool);
1402	if (index == -1) {
1403	if (!(attrs & DMA_ATTR_NO_WARN))
1404	dev_warn_ratelimited(dev,
1405	"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1406	size, mem->nslabs, mem_used(mem));
1407	return (phys_addr_t)DMA_MAPPING_ERROR;
1408	}
1409
1410	/*
1411	* If dma_skip_sync was set, reset it on first SWIOTLB buffer
1412	* mapping to always sync SWIOTLB buffers.
1413	*/
1414	dma_reset_need_sync(dev);
1415
1416	/*
1417	* Save away the mapping from the original address to the DMA address.
1418	* This is needed when we sync the memory. Then we sync the buffer if
1419	* needed.
1420	*/
1421	pad_slots = offset >> IO_TLB_SHIFT;
1422	offset &= (IO_TLB_SIZE - 1);
1423	index += pad_slots;
1424	pool->slots[index].pad_slots = pad_slots;
1425	for (i = 0; i < (nr_slots(val: size) - pad_slots); i++)
1426	pool->slots[index + i].orig_addr = slot_addr(start: orig_addr, idx: i);
1427	tlb_addr = slot_addr(start: pool->start, idx: index) + offset;
1428	/*
1429	* When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1430	* the original buffer to the TLB buffer before initiating DMA in order
1431	* to preserve the original's data if the device does a partial write,
1432	* i.e. if the device doesn't overwrite the entire buffer. Preserving
1433	* the original data, even if it's garbage, is necessary to match
1434	* hardware behavior. Use of swiotlb is supposed to be transparent,
1435	* i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1436	*/
1437	swiotlb_bounce(dev, tlb_addr, size: mapping_size, dir: DMA_TO_DEVICE, mem: pool);
1438	return tlb_addr;
1439	}
1440
1441	static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr,
1442	struct io_tlb_pool *mem)
1443	{
1444	unsigned long flags;
1445	unsigned int offset = swiotlb_align_offset(dev, align_mask: 0, addr: tlb_addr);
1446	int index, nslots, aindex;
1447	struct io_tlb_area *area;
1448	int count, i;
1449
1450	index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1451	index -= mem->slots[index].pad_slots;
1452	nslots = nr_slots(val: mem->slots[index].alloc_size + offset);
1453	aindex = index / mem->area_nslabs;
1454	area = &mem->areas[aindex];
1455
1456	/*
1457	* Return the buffer to the free list by setting the corresponding
1458	* entries to indicate the number of contiguous entries available.
1459	* While returning the entries to the free list, we merge the entries
1460	* with slots below and above the pool being returned.
1461	*/
1462	BUG_ON(aindex >= mem->nareas);
1463
1464	spin_lock_irqsave(&area->lock, flags);
1465	if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1466	count = mem->slots[index + nslots].list;
1467	else
1468	count = 0;
1469
1470	/*
1471	* Step 1: return the slots to the free list, merging the slots with
1472	* superceeding slots
1473	*/
1474	for (i = index + nslots - 1; i >= index; i--) {
1475	mem->slots[i].list = ++count;
1476	mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1477	mem->slots[i].alloc_size = 0;
1478	mem->slots[i].pad_slots = 0;
1479	}
1480
1481	/*
1482	* Step 2: merge the returned slots with the preceding slots, if
1483	* available (non zero)
1484	*/
1485	for (i = index - 1;
1486	io_tlb_offset(val: i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1487	i--)
1488	mem->slots[i].list = ++count;
1489	area->used -= nslots;
1490	spin_unlock_irqrestore(lock: &area->lock, flags);
1491
1492	dec_used(mem: dev->dma_io_tlb_mem, nslots);
1493	}
1494
1495	#ifdef CONFIG_SWIOTLB_DYNAMIC
1496
1497	/**
1498	* swiotlb_del_transient() - delete a transient memory pool
1499	* @dev: Device which mapped the buffer.
1500	* @tlb_addr: Physical address within a bounce buffer.
1501	* @pool: Pointer to the transient memory pool to be checked and deleted.
1502	*
1503	* Check whether the address belongs to a transient SWIOTLB memory pool.
1504	* If yes, then delete the pool.
1505	*
1506	* Return: %true if @tlb_addr belonged to a transient pool that was released.
1507	*/
1508	static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr,
1509	struct io_tlb_pool *pool)
1510	{
1511	if (!pool->transient)
1512	return false;
1513
1514	dec_used(mem: dev->dma_io_tlb_mem, nslots: pool->nslabs);
1515	swiotlb_del_pool(dev, pool);
1516	dec_transient_used(mem: dev->dma_io_tlb_mem, nslots: pool->nslabs);
1517	return true;
1518	}
1519
1520	#else /* !CONFIG_SWIOTLB_DYNAMIC */
1521
1522	static inline bool swiotlb_del_transient(struct device *dev,
1523	phys_addr_t tlb_addr, struct io_tlb_pool *pool)
1524	{
1525	return false;
1526	}
1527
1528	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1529
1530	/*
1531	* tlb_addr is the physical address of the bounce buffer to unmap.
1532	*/
1533	void __swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1534	size_t mapping_size, enum dma_data_direction dir,
1535	unsigned long attrs, struct io_tlb_pool *pool)
1536	{
1537	/*
1538	* First, sync the memory before unmapping the entry
1539	*/
1540	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1541	(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1542	swiotlb_bounce(dev, tlb_addr, size: mapping_size,
1543	dir: DMA_FROM_DEVICE, mem: pool);
1544
1545	if (swiotlb_del_transient(dev, tlb_addr, pool))
1546	return;
1547	swiotlb_release_slots(dev, tlb_addr, mem: pool);
1548	}
1549
1550	void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1551	size_t size, enum dma_data_direction dir,
1552	struct io_tlb_pool *pool)
1553	{
1554	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1555	swiotlb_bounce(dev, tlb_addr, size, dir: DMA_TO_DEVICE, mem: pool);
1556	else
1557	BUG_ON(dir != DMA_FROM_DEVICE);
1558	}
1559
1560	void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1561	size_t size, enum dma_data_direction dir,
1562	struct io_tlb_pool *pool)
1563	{
1564	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1565	swiotlb_bounce(dev, tlb_addr, size, dir: DMA_FROM_DEVICE, mem: pool);
1566	else
1567	BUG_ON(dir != DMA_TO_DEVICE);
1568	}
1569
1570	/*
1571	* Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1572	* to the device copy the data into it as well.
1573	*/
1574	dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1575	enum dma_data_direction dir, unsigned long attrs)
1576	{
1577	phys_addr_t swiotlb_addr;
1578	dma_addr_t dma_addr;
1579
1580	trace_swiotlb_bounced(dev, dev_addr: phys_to_dma(dev, paddr), size);
1581
1582	swiotlb_addr = swiotlb_tbl_map_single(dev, orig_addr: paddr, mapping_size: size, alloc_align_mask: 0, dir, attrs);
1583	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1584	return DMA_MAPPING_ERROR;
1585
1586	/* Ensure that the address returned is DMA'ble */
1587	dma_addr = phys_to_dma_unencrypted(dev, paddr: swiotlb_addr);
1588	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1589	__swiotlb_tbl_unmap_single(dev, tlb_addr: swiotlb_addr, mapping_size: size, dir,
1590	attrs: attrs | DMA_ATTR_SKIP_CPU_SYNC,
1591	pool: swiotlb_find_pool(dev, paddr: swiotlb_addr));
1592	dev_WARN_ONCE(dev, 1,
1593	"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1594	&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1595	return DMA_MAPPING_ERROR;
1596	}
1597
1598	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1599	arch_sync_dma_for_device(paddr: swiotlb_addr, size, dir);
1600	return dma_addr;
1601	}
1602
1603	size_t swiotlb_max_mapping_size(struct device *dev)
1604	{
1605	int min_align_mask = dma_get_min_align_mask(dev);
1606	int min_align = 0;
1607
1608	/*
1609	* swiotlb_find_slots() skips slots according to
1610	* min align mask. This affects max mapping size.
1611	* Take it into acount here.
1612	*/
1613	if (min_align_mask)
1614	min_align = roundup(min_align_mask, IO_TLB_SIZE);
1615
1616	return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1617	}
1618
1619	/**
1620	* is_swiotlb_allocated() - check if the default software IO TLB is initialized
1621	*/
1622	bool is_swiotlb_allocated(void)
1623	{
1624	return io_tlb_default_mem.nslabs;
1625	}
1626
1627	bool is_swiotlb_active(struct device *dev)
1628	{
1629	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1630
1631	return mem && mem->nslabs;
1632	}
1633
1634	/**
1635	* default_swiotlb_base() - get the base address of the default SWIOTLB
1636	*
1637	* Get the lowest physical address used by the default software IO TLB pool.
1638	*/
1639	phys_addr_t default_swiotlb_base(void)
1640	{
1641	#ifdef CONFIG_SWIOTLB_DYNAMIC
1642	io_tlb_default_mem.can_grow = false;
1643	#endif
1644	return io_tlb_default_mem.defpool.start;
1645	}
1646
1647	/**
1648	* default_swiotlb_limit() - get the address limit of the default SWIOTLB
1649	*
1650	* Get the highest physical address used by the default software IO TLB pool.
1651	*/
1652	phys_addr_t default_swiotlb_limit(void)
1653	{
1654	#ifdef CONFIG_SWIOTLB_DYNAMIC
1655	return io_tlb_default_mem.phys_limit;
1656	#else
1657	return io_tlb_default_mem.defpool.end - 1;
1658	#endif
1659	}
1660
1661	#ifdef CONFIG_DEBUG_FS
1662	#ifdef CONFIG_SWIOTLB_DYNAMIC
1663	static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1664	{
1665	return atomic_long_read(v: &mem->transient_nslabs);
1666	}
1667
1668	static int io_tlb_transient_used_get(void *data, u64 *val)
1669	{
1670	struct io_tlb_mem *mem = data;
1671
1672	*val = mem_transient_used(mem);
1673	return 0;
1674	}
1675
1676	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1677	NULL, "%llu\n");
1678	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1679
1680	static int io_tlb_used_get(void *data, u64 *val)
1681	{
1682	struct io_tlb_mem *mem = data;
1683
1684	*val = mem_used(mem);
1685	return 0;
1686	}
1687
1688	static int io_tlb_hiwater_get(void *data, u64 *val)
1689	{
1690	struct io_tlb_mem *mem = data;
1691
1692	*val = atomic_long_read(v: &mem->used_hiwater);
1693	return 0;
1694	}
1695
1696	static int io_tlb_hiwater_set(void *data, u64 val)
1697	{
1698	struct io_tlb_mem *mem = data;
1699
1700	/* Only allow setting to zero */
1701	if (val != 0)
1702	return -EINVAL;
1703
1704	atomic_long_set(v: &mem->used_hiwater, i: val);
1705	return 0;
1706	}
1707
1708	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1709	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1710	io_tlb_hiwater_set, "%llu\n");
1711
1712	static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1713	const char *dirname)
1714	{
1715	mem->debugfs = debugfs_create_dir(name: dirname, parent: io_tlb_default_mem.debugfs);
1716	if (!mem->nslabs)
1717	return;
1718
1719	debugfs_create_ulong(name: "io_tlb_nslabs", mode: 0400, parent: mem->debugfs, value: &mem->nslabs);
1720	debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
1721	&fops_io_tlb_used);
1722	debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
1723	&fops_io_tlb_hiwater);
1724	#ifdef CONFIG_SWIOTLB_DYNAMIC
1725	debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
1726	mem, &fops_io_tlb_transient_used);
1727	#endif
1728	}
1729
1730	static int __init swiotlb_create_default_debugfs(void)
1731	{
1732	swiotlb_create_debugfs_files(mem: &io_tlb_default_mem, dirname: "swiotlb");
1733	return 0;
1734	}
1735
1736	late_initcall(swiotlb_create_default_debugfs);
1737
1738	#else /* !CONFIG_DEBUG_FS */
1739
1740	static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1741	const char *dirname)
1742	{
1743	}
1744
1745	#endif /* CONFIG_DEBUG_FS */
1746
1747	#ifdef CONFIG_DMA_RESTRICTED_POOL
1748
1749	struct page *swiotlb_alloc(struct device *dev, size_t size)
1750	{
1751	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1752	struct io_tlb_pool *pool;
1753	phys_addr_t tlb_addr;
1754	unsigned int align;
1755	int index;
1756
1757	if (!mem)
1758	return NULL;
1759
1760	align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1761	index = swiotlb_find_slots(dev, orig_addr: 0, alloc_size: size, alloc_align_mask: align, retpool: &pool);
1762	if (index == -1)
1763	return NULL;
1764
1765	tlb_addr = slot_addr(start: pool->start, idx: index);
1766	if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1767	dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1768	&tlb_addr);
1769	swiotlb_release_slots(dev, tlb_addr, mem: pool);
1770	return NULL;
1771	}
1772
1773	return pfn_to_page(PFN_DOWN(tlb_addr));
1774	}
1775
1776	bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1777	{
1778	phys_addr_t tlb_addr = page_to_phys(page);
1779	struct io_tlb_pool *pool;
1780
1781	pool = swiotlb_find_pool(dev, paddr: tlb_addr);
1782	if (!pool)
1783	return false;
1784
1785	swiotlb_release_slots(dev, tlb_addr, mem: pool);
1786
1787	return true;
1788	}
1789
1790	static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1791	struct device *dev)
1792	{
1793	struct io_tlb_mem *mem = rmem->priv;
1794	unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1795
1796	/* Set Per-device io tlb area to one */
1797	unsigned int nareas = 1;
1798
1799	if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1800	dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1801	return -EINVAL;
1802	}
1803
1804	/*
1805	* Since multiple devices can share the same pool, the private data,
1806	* io_tlb_mem struct, will be initialized by the first device attached
1807	* to it.
1808	*/
1809	if (!mem) {
1810	struct io_tlb_pool *pool;
1811
1812	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
1813	if (!mem)
1814	return -ENOMEM;
1815	pool = &mem->defpool;
1816
1817	pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
1818	if (!pool->slots) {
1819	kfree(objp: mem);
1820	return -ENOMEM;
1821	}
1822
1823	pool->areas = kcalloc(nareas, sizeof(*pool->areas),
1824	GFP_KERNEL);
1825	if (!pool->areas) {
1826	kfree(objp: pool->slots);
1827	kfree(objp: mem);
1828	return -ENOMEM;
1829	}
1830
1831	set_memory_decrypted(addr: (unsigned long)phys_to_virt(address: rmem->base),
1832	numpages: rmem->size >> PAGE_SHIFT);
1833	swiotlb_init_io_tlb_pool(mem: pool, start: rmem->base, nslabs,
1834	late_alloc: false, nareas);
1835	mem->force_bounce = true;
1836	mem->for_alloc = true;
1837	#ifdef CONFIG_SWIOTLB_DYNAMIC
1838	spin_lock_init(&mem->lock);
1839	INIT_LIST_HEAD_RCU(list: &mem->pools);
1840	#endif
1841	add_mem_pool(mem, pool);
1842
1843	rmem->priv = mem;
1844
1845	swiotlb_create_debugfs_files(mem, dirname: rmem->name);
1846	}
1847
1848	dev->dma_io_tlb_mem = mem;
1849
1850	return 0;
1851	}
1852
1853	static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1854	struct device *dev)
1855	{
1856	dev->dma_io_tlb_mem = &io_tlb_default_mem;
1857	}
1858
1859	static const struct reserved_mem_ops rmem_swiotlb_ops = {
1860	.device_init = rmem_swiotlb_device_init,
1861	.device_release = rmem_swiotlb_device_release,
1862	};
1863
1864	static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1865	{
1866	unsigned long node = rmem->fdt_node;
1867
1868	if (of_get_flat_dt_prop(node, name: "reusable", NULL) ||
1869	of_get_flat_dt_prop(node, name: "linux,cma-default", NULL) ||
1870	of_get_flat_dt_prop(node, name: "linux,dma-default", NULL) ||
1871	of_get_flat_dt_prop(node, name: "no-map", NULL))
1872	return -EINVAL;
1873
1874	rmem->ops = &rmem_swiotlb_ops;
1875	pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1876	&rmem->base, (unsigned long)rmem->size / SZ_1M);
1877	return 0;
1878	}
1879
1880	RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1881	#endif /* CONFIG_DMA_RESTRICTED_POOL */
1882