1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Dynamic DMA mapping support.
4	*
5	* This implementation is a fallback for platforms that do not support
6	* I/O TLBs (aka DMA address translation hardware).
7	* Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
8	* Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
9	* Copyright (C) 2000, 2003 Hewlett-Packard Co
10	* David Mosberger-Tang <davidm@hpl.hp.com>
11	*
12	* 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13	* 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14	* unnecessary i-cache flushing.
15	* 04/07/.. ak Better overflow handling. Assorted fixes.
16	* 05/09/10 linville Add support for syncing ranges, support syncing for
17	* DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18	* 08/12/11 beckyb Add highmem support
19	*/
20
21	#define pr_fmt(fmt) "software IO TLB: " fmt
22
23	#include <linux/cache.h>
24	#include <linux/cc_platform.h>
25	#include <linux/ctype.h>
26	#include <linux/debugfs.h>
27	#include <linux/dma-direct.h>
28	#include <linux/dma-map-ops.h>
29	#include <linux/export.h>
30	#include <linux/gfp.h>
31	#include <linux/highmem.h>
32	#include <linux/io.h>
33	#include <linux/iommu-helper.h>
34	#include <linux/init.h>
35	#include <linux/memblock.h>
36	#include <linux/mm.h>
37	#include <linux/pfn.h>
38	#include <linux/rculist.h>
39	#include <linux/scatterlist.h>
40	#include <linux/set_memory.h>
41	#include <linux/spinlock.h>
42	#include <linux/string.h>
43	#include <linux/swiotlb.h>
44	#include <linux/types.h>
45	#ifdef CONFIG_DMA_RESTRICTED_POOL
46	#include <linux/of.h>
47	#include <linux/of_fdt.h>
48	#include <linux/of_reserved_mem.h>
49	#include <linux/slab.h>
50	#endif
51
52	#define CREATE_TRACE_POINTS
53	#include <trace/events/swiotlb.h>
54
55	#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
56
57	/*
58	* Minimum IO TLB size to bother booting with. Systems with mainly
59	* 64bit capable cards will only lightly use the swiotlb. If we can't
60	* allocate a contiguous 1MB, we're probably in trouble anyway.
61	*/
62	#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
63
64	#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
65
66	/**
67	* struct io_tlb_slot - IO TLB slot descriptor
68	* @orig_addr: The original address corresponding to a mapped entry.
69	* @alloc_size: Size of the allocated buffer.
70	* @list: The free list describing the number of free entries available
71	* from each index.
72	* @pad_slots: Number of preceding padding slots. Valid only in the first
73	* allocated non-padding slot.
74	*/
75	struct io_tlb_slot {
76	phys_addr_t orig_addr;
77	size_t alloc_size;
78	unsigned short list;
79	unsigned short pad_slots;
80	};
81
82	static bool swiotlb_force_bounce;
83	static bool swiotlb_force_disable;
84
85	#ifdef CONFIG_SWIOTLB_DYNAMIC
86
87	static void swiotlb_dyn_alloc(struct work_struct *work);
88
89	static struct io_tlb_mem io_tlb_default_mem = {
90	.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
91	.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
92	.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
93	swiotlb_dyn_alloc),
94	};
95
96	#else /* !CONFIG_SWIOTLB_DYNAMIC */
97
98	static struct io_tlb_mem io_tlb_default_mem;
99
100	#endif /* CONFIG_SWIOTLB_DYNAMIC */
101
102	static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
103	static unsigned long default_nareas;
104
105	/**
106	* struct io_tlb_area - IO TLB memory area descriptor
107	*
108	* This is a single area with a single lock.
109	*
110	* @used: The number of used IO TLB block.
111	* @index: The slot index to start searching in this area for next round.
112	* @lock: The lock to protect the above data structures in the map and
113	* unmap calls.
114	*/
115	struct io_tlb_area {
116	unsigned long used;
117	unsigned int index;
118	spinlock_t lock;
119	};
120
121	/*
122	* Round up number of slabs to the next power of 2. The last area is going
123	* be smaller than the rest if default_nslabs is not power of two.
124	* The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
125	* otherwise a segment may span two or more areas. It conflicts with free
126	* contiguous slots tracking: free slots are treated contiguous no matter
127	* whether they cross an area boundary.
128	*
129	* Return true if default_nslabs is rounded up.
130	*/
131	static bool round_up_default_nslabs(void)
132	{
133	if (!default_nareas)
134	return false;
135
136	if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
137	default_nslabs = IO_TLB_SEGSIZE * default_nareas;
138	else if (is_power_of_2(n: default_nslabs))
139	return false;
140	default_nslabs = roundup_pow_of_two(default_nslabs);
141	return true;
142	}
143
144	/**
145	* swiotlb_adjust_nareas() - adjust the number of areas and slots
146	* @nareas: Desired number of areas. Zero is treated as 1.
147	*
148	* Adjust the default number of areas in a memory pool.
149	* The default size of the memory pool may also change to meet minimum area
150	* size requirements.
151	*/
152	static void swiotlb_adjust_nareas(unsigned int nareas)
153	{
154	if (!nareas)
155	nareas = 1;
156	else if (!is_power_of_2(n: nareas))
157	nareas = roundup_pow_of_two(nareas);
158
159	default_nareas = nareas;
160
161	pr_info("area num %d.\n", nareas);
162	if (round_up_default_nslabs())
163	pr_info("SWIOTLB bounce buffer size roundup to %luMB",
164	(default_nslabs << IO_TLB_SHIFT) >> 20);
165	}
166
167	/**
168	* limit_nareas() - get the maximum number of areas for a given memory pool size
169	* @nareas: Desired number of areas.
170	* @nslots: Total number of slots in the memory pool.
171	*
172	* Limit the number of areas to the maximum possible number of areas in
173	* a memory pool of the given size.
174	*
175	* Return: Maximum possible number of areas.
176	*/
177	static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
178	{
179	if (nslots < nareas * IO_TLB_SEGSIZE)
180	return nslots / IO_TLB_SEGSIZE;
181	return nareas;
182	}
183
184	static int __init
185	setup_io_tlb_npages(char *str)
186	{
187	if (isdigit(c: *str)) {
188	/* avoid tail segment of size < IO_TLB_SEGSIZE */
189	default_nslabs =
190	ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
191	}
192	if (*str == ',')
193	++str;
194	if (isdigit(c: *str))
195	swiotlb_adjust_nareas(nareas: simple_strtoul(str, &str, 0));
196	if (*str == ',')
197	++str;
198	if (!strcmp(str, "force"))
199	swiotlb_force_bounce = true;
200	else if (!strcmp(str, "noforce"))
201	swiotlb_force_disable = true;
202
203	return 0;
204	}
205	early_param("swiotlb", setup_io_tlb_npages);
206
207	unsigned long swiotlb_size_or_default(void)
208	{
209	return default_nslabs << IO_TLB_SHIFT;
210	}
211
212	void __init swiotlb_adjust_size(unsigned long size)
213	{
214	/*
215	* If swiotlb parameter has not been specified, give a chance to
216	* architectures such as those supporting memory encryption to
217	* adjust/expand SWIOTLB size for their use.
218	*/
219	if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
220	return;
221
222	size = ALIGN(size, IO_TLB_SIZE);
223	default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
224	if (round_up_default_nslabs())
225	size = default_nslabs << IO_TLB_SHIFT;
226	pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
227	}
228
229	void swiotlb_print_info(void)
230	{
231	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
232
233	if (!mem->nslabs) {
234	pr_warn("No low mem\n");
235	return;
236	}
237
238	pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
239	(mem->nslabs << IO_TLB_SHIFT) >> 20);
240	}
241
242	static inline unsigned long io_tlb_offset(unsigned long val)
243	{
244	return val & (IO_TLB_SEGSIZE - 1);
245	}
246
247	static inline unsigned long nr_slots(u64 val)
248	{
249	return DIV_ROUND_UP(val, IO_TLB_SIZE);
250	}
251
252	/*
253	* Early SWIOTLB allocation may be too early to allow an architecture to
254	* perform the desired operations. This function allows the architecture to
255	* call SWIOTLB when the operations are possible. It needs to be called
256	* before the SWIOTLB memory is used.
257	*/
258	void __init swiotlb_update_mem_attributes(void)
259	{
260	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
261	unsigned long bytes;
262
263	if (!mem->nslabs || mem->late_alloc)
264	return;
265	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
266	set_memory_decrypted(addr: (unsigned long)mem->vaddr, numpages: bytes >> PAGE_SHIFT);
267	}
268
269	static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
270	unsigned long nslabs, bool late_alloc, unsigned int nareas)
271	{
272	void *vaddr = phys_to_virt(address: start);
273	unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
274
275	mem->nslabs = nslabs;
276	mem->start = start;
277	mem->end = mem->start + bytes;
278	mem->late_alloc = late_alloc;
279	mem->nareas = nareas;
280	mem->area_nslabs = nslabs / mem->nareas;
281
282	for (i = 0; i < mem->nareas; i++) {
283	spin_lock_init(&mem->areas[i].lock);
284	mem->areas[i].index = 0;
285	mem->areas[i].used = 0;
286	}
287
288	for (i = 0; i < mem->nslabs; i++) {
289	mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i),
290	mem->nslabs - i);
291	mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
292	mem->slots[i].alloc_size = 0;
293	mem->slots[i].pad_slots = 0;
294	}
295
296	memset(vaddr, 0, bytes);
297	mem->vaddr = vaddr;
298	return;
299	}
300
301	/**
302	* add_mem_pool() - add a memory pool to the allocator
303	* @mem: Software IO TLB allocator.
304	* @pool: Memory pool to be added.
305	*/
306	static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
307	{
308	#ifdef CONFIG_SWIOTLB_DYNAMIC
309	spin_lock(lock: &mem->lock);
310	list_add_rcu(new: &pool->node, head: &mem->pools);
311	mem->nslabs += pool->nslabs;
312	spin_unlock(lock: &mem->lock);
313	#else
314	mem->nslabs = pool->nslabs;
315	#endif
316	}
317
318	static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
319	unsigned int flags,
320	int (*remap)(void *tlb, unsigned long nslabs))
321	{
322	size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
323	void *tlb;
324
325	/*
326	* By default allocate the bounce buffer memory from low memory, but
327	* allow to pick a location everywhere for hypervisors with guest
328	* memory encryption.
329	*/
330	if (flags & SWIOTLB_ANY)
331	tlb = memblock_alloc(size: bytes, PAGE_SIZE);
332	else
333	tlb = memblock_alloc_low(size: bytes, PAGE_SIZE);
334
335	if (!tlb) {
336	pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
337	__func__, bytes);
338	return NULL;
339	}
340
341	if (remap && remap(tlb, nslabs) < 0) {
342	memblock_free(ptr: tlb, PAGE_ALIGN(bytes));
343	pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
344	return NULL;
345	}
346
347	return tlb;
348	}
349
350	/*
351	* Statically reserve bounce buffer space and initialize bounce buffer data
352	* structures for the software IO TLB used to implement the DMA API.
353	*/
354	void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
355	int (*remap)(void *tlb, unsigned long nslabs))
356	{
357	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
358	unsigned long nslabs;
359	unsigned int nareas;
360	size_t alloc_size;
361	void *tlb;
362
363	if (!addressing_limit && !swiotlb_force_bounce)
364	return;
365	if (swiotlb_force_disable)
366	return;
367
368	io_tlb_default_mem.force_bounce =
369	swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
370
371	#ifdef CONFIG_SWIOTLB_DYNAMIC
372	if (!remap)
373	io_tlb_default_mem.can_grow = true;
374	if (flags & SWIOTLB_ANY)
375	io_tlb_default_mem.phys_limit = virt_to_phys(address: high_memory - 1);
376	else
377	io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
378	#endif
379
380	if (!default_nareas)
381	swiotlb_adjust_nareas(num_possible_cpus());
382
383	nslabs = default_nslabs;
384	nareas = limit_nareas(nareas: default_nareas, nslots: nslabs);
385	while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
386	if (nslabs <= IO_TLB_MIN_SLABS)
387	return;
388	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
389	nareas = limit_nareas(nareas, nslots: nslabs);
390	}
391
392	if (default_nslabs != nslabs) {
393	pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
394	default_nslabs, nslabs);
395	default_nslabs = nslabs;
396	}
397
398	alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
399	mem->slots = memblock_alloc(size: alloc_size, PAGE_SIZE);
400	if (!mem->slots) {
401	pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
402	__func__, alloc_size, PAGE_SIZE);
403	return;
404	}
405
406	mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
407	nareas), SMP_CACHE_BYTES);
408	if (!mem->areas) {
409	pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
410	return;
411	}
412
413	swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, late_alloc: false, nareas);
414	add_mem_pool(mem: &io_tlb_default_mem, pool: mem);
415
416	if (flags & SWIOTLB_VERBOSE)
417	swiotlb_print_info();
418	}
419
420	void __init swiotlb_init(bool addressing_limit, unsigned int flags)
421	{
422	swiotlb_init_remap(addressing_limit, flags, NULL);
423	}
424
425	/*
426	* Systems with larger DMA zones (those that don't support ISA) can
427	* initialize the swiotlb later using the slab allocator if needed.
428	* This should be just like above, but with some error catching.
429	*/
430	int swiotlb_init_late(size_t size, gfp_t gfp_mask,
431	int (*remap)(void *tlb, unsigned long nslabs))
432	{
433	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
434	unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
435	unsigned int nareas;
436	unsigned char *vstart = NULL;
437	unsigned int order, area_order;
438	bool retried = false;
439	int rc = 0;
440
441	if (io_tlb_default_mem.nslabs)
442	return 0;
443
444	if (swiotlb_force_disable)
445	return 0;
446
447	io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
448
449	#ifdef CONFIG_SWIOTLB_DYNAMIC
450	if (!remap)
451	io_tlb_default_mem.can_grow = true;
452	if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
453	io_tlb_default_mem.phys_limit = DMA_BIT_MASK(zone_dma_bits);
454	else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
455	io_tlb_default_mem.phys_limit = DMA_BIT_MASK(32);
456	else
457	io_tlb_default_mem.phys_limit = virt_to_phys(address: high_memory - 1);
458	#endif
459
460	if (!default_nareas)
461	swiotlb_adjust_nareas(num_possible_cpus());
462
463	retry:
464	order = get_order(size: nslabs << IO_TLB_SHIFT);
465	nslabs = SLABS_PER_PAGE << order;
466
467	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
468	vstart = (void *)__get_free_pages(gfp_mask: gfp_mask | __GFP_NOWARN,
469	order);
470	if (vstart)
471	break;
472	order--;
473	nslabs = SLABS_PER_PAGE << order;
474	retried = true;
475	}
476
477	if (!vstart)
478	return -ENOMEM;
479
480	if (remap)
481	rc = remap(vstart, nslabs);
482	if (rc) {
483	free_pages(addr: (unsigned long)vstart, order);
484
485	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
486	if (nslabs < IO_TLB_MIN_SLABS)
487	return rc;
488	retried = true;
489	goto retry;
490	}
491
492	if (retried) {
493	pr_warn("only able to allocate %ld MB\n",
494	(PAGE_SIZE << order) >> 20);
495	}
496
497	nareas = limit_nareas(nareas: default_nareas, nslots: nslabs);
498	area_order = get_order(array_size(sizeof(*mem->areas), nareas));
499	mem->areas = (struct io_tlb_area *)
500	__get_free_pages(GFP_KERNEL | __GFP_ZERO, order: area_order);
501	if (!mem->areas)
502	goto error_area;
503
504	mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
505	order: get_order(array_size(sizeof(*mem->slots), nslabs)));
506	if (!mem->slots)
507	goto error_slots;
508
509	set_memory_decrypted(addr: (unsigned long)vstart,
510	numpages: (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
511	swiotlb_init_io_tlb_pool(mem, virt_to_phys(address: vstart), nslabs, late_alloc: true,
512	nareas);
513	add_mem_pool(mem: &io_tlb_default_mem, pool: mem);
514
515	swiotlb_print_info();
516	return 0;
517
518	error_slots:
519	free_pages(addr: (unsigned long)mem->areas, order: area_order);
520	error_area:
521	free_pages(addr: (unsigned long)vstart, order);
522	return -ENOMEM;
523	}
524
525	void __init swiotlb_exit(void)
526	{
527	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
528	unsigned long tbl_vaddr;
529	size_t tbl_size, slots_size;
530	unsigned int area_order;
531
532	if (swiotlb_force_bounce)
533	return;
534
535	if (!mem->nslabs)
536	return;
537
538	pr_info("tearing down default memory pool\n");
539	tbl_vaddr = (unsigned long)phys_to_virt(address: mem->start);
540	tbl_size = PAGE_ALIGN(mem->end - mem->start);
541	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
542
543	set_memory_encrypted(addr: tbl_vaddr, numpages: tbl_size >> PAGE_SHIFT);
544	if (mem->late_alloc) {
545	area_order = get_order(array_size(sizeof(*mem->areas),
546	mem->nareas));
547	free_pages(addr: (unsigned long)mem->areas, order: area_order);
548	free_pages(addr: tbl_vaddr, order: get_order(size: tbl_size));
549	free_pages(addr: (unsigned long)mem->slots, order: get_order(size: slots_size));
550	} else {
551	memblock_free_late(__pa(mem->areas),
552	array_size(sizeof(*mem->areas), mem->nareas));
553	memblock_free_late(base: mem->start, size: tbl_size);
554	memblock_free_late(__pa(mem->slots), size: slots_size);
555	}
556
557	memset(mem, 0, sizeof(*mem));
558	}
559
560	#ifdef CONFIG_SWIOTLB_DYNAMIC
561
562	/**
563	* alloc_dma_pages() - allocate pages to be used for DMA
564	* @gfp: GFP flags for the allocation.
565	* @bytes: Size of the buffer.
566	* @phys_limit: Maximum allowed physical address of the buffer.
567	*
568	* Allocate pages from the buddy allocator. If successful, make the allocated
569	* pages decrypted that they can be used for DMA.
570	*
571	* Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
572	* if the allocated physical address was above @phys_limit.
573	*/
574	static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
575	{
576	unsigned int order = get_order(size: bytes);
577	struct page *page;
578	phys_addr_t paddr;
579	void *vaddr;
580
581	page = alloc_pages(gfp, order);
582	if (!page)
583	return NULL;
584
585	paddr = page_to_phys(page);
586	if (paddr + bytes - 1 > phys_limit) {
587	__free_pages(page, order);
588	return ERR_PTR(error: -EAGAIN);
589	}
590
591	vaddr = phys_to_virt(address: paddr);
592	if (set_memory_decrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
593	goto error;
594	return page;
595
596	error:
597	/* Intentional leak if pages cannot be encrypted again. */
598	if (!set_memory_encrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
599	__free_pages(page, order);
600	return NULL;
601	}
602
603	/**
604	* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
605	* @dev: Device for which a memory pool is allocated.
606	* @bytes: Size of the buffer.
607	* @phys_limit: Maximum allowed physical address of the buffer.
608	* @gfp: GFP flags for the allocation.
609	*
610	* Return: Allocated pages, or %NULL on allocation failure.
611	*/
612	static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
613	u64 phys_limit, gfp_t gfp)
614	{
615	struct page *page;
616
617	/*
618	* Allocate from the atomic pools if memory is encrypted and
619	* the allocation is atomic, because decrypting may block.
620	*/
621	if (!gfpflags_allow_blocking(gfp_flags: gfp) && dev && force_dma_unencrypted(dev)) {
622	void *vaddr;
623
624	if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
625	return NULL;
626
627	return dma_alloc_from_pool(dev, size: bytes, cpu_addr: &vaddr, flags: gfp,
628	phys_addr_ok: dma_coherent_ok);
629	}
630
631	gfp &= ~GFP_ZONEMASK;
632	if (phys_limit <= DMA_BIT_MASK(zone_dma_bits))
633	gfp |= __GFP_DMA;
634	else if (phys_limit <= DMA_BIT_MASK(32))
635	gfp |= __GFP_DMA32;
636
637	while (IS_ERR(ptr: page = alloc_dma_pages(gfp, bytes, phys_limit))) {
638	if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
639	phys_limit < DMA_BIT_MASK(64) &&
640	!(gfp & (__GFP_DMA32 | __GFP_DMA)))
641	gfp |= __GFP_DMA32;
642	else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
643	!(gfp & __GFP_DMA))
644	gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
645	else
646	return NULL;
647	}
648
649	return page;
650	}
651
652	/**
653	* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
654	* @vaddr: Virtual address of the buffer.
655	* @bytes: Size of the buffer.
656	*/
657	static void swiotlb_free_tlb(void *vaddr, size_t bytes)
658	{
659	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
660	dma_free_from_pool(NULL, start: vaddr, size: bytes))
661	return;
662
663	/* Intentional leak if pages cannot be encrypted again. */
664	if (!set_memory_encrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
665	__free_pages(virt_to_page(vaddr), order: get_order(size: bytes));
666	}
667
668	/**
669	* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
670	* @dev: Device for which a memory pool is allocated.
671	* @minslabs: Minimum number of slabs.
672	* @nslabs: Desired (maximum) number of slabs.
673	* @nareas: Number of areas.
674	* @phys_limit: Maximum DMA buffer physical address.
675	* @gfp: GFP flags for the allocations.
676	*
677	* Allocate and initialize a new IO TLB memory pool. The actual number of
678	* slabs may be reduced if allocation of @nslabs fails. If even
679	* @minslabs cannot be allocated, this function fails.
680	*
681	* Return: New memory pool, or %NULL on allocation failure.
682	*/
683	static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
684	unsigned long minslabs, unsigned long nslabs,
685	unsigned int nareas, u64 phys_limit, gfp_t gfp)
686	{
687	struct io_tlb_pool *pool;
688	unsigned int slot_order;
689	struct page *tlb;
690	size_t pool_size;
691	size_t tlb_size;
692
693	if (nslabs > SLABS_PER_PAGE << MAX_PAGE_ORDER) {
694	nslabs = SLABS_PER_PAGE << MAX_PAGE_ORDER;
695	nareas = limit_nareas(nareas, nslots: nslabs);
696	}
697
698	pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
699	pool = kzalloc(size: pool_size, flags: gfp);
700	if (!pool)
701	goto error;
702	pool->areas = (void *)pool + sizeof(*pool);
703
704	tlb_size = nslabs << IO_TLB_SHIFT;
705	while (!(tlb = swiotlb_alloc_tlb(dev, bytes: tlb_size, phys_limit, gfp))) {
706	if (nslabs <= minslabs)
707	goto error_tlb;
708	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
709	nareas = limit_nareas(nareas, nslots: nslabs);
710	tlb_size = nslabs << IO_TLB_SHIFT;
711	}
712
713	slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
714	pool->slots = (struct io_tlb_slot *)
715	__get_free_pages(gfp_mask: gfp, order: slot_order);
716	if (!pool->slots)
717	goto error_slots;
718
719	swiotlb_init_io_tlb_pool(mem: pool, page_to_phys(tlb), nslabs, late_alloc: true, nareas);
720	return pool;
721
722	error_slots:
723	swiotlb_free_tlb(page_address(tlb), bytes: tlb_size);
724	error_tlb:
725	kfree(objp: pool);
726	error:
727	return NULL;
728	}
729
730	/**
731	* swiotlb_dyn_alloc() - dynamic memory pool allocation worker
732	* @work: Pointer to dyn_alloc in struct io_tlb_mem.
733	*/
734	static void swiotlb_dyn_alloc(struct work_struct *work)
735	{
736	struct io_tlb_mem *mem =
737	container_of(work, struct io_tlb_mem, dyn_alloc);
738	struct io_tlb_pool *pool;
739
740	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, nslabs: default_nslabs,
741	nareas: default_nareas, phys_limit: mem->phys_limit, GFP_KERNEL);
742	if (!pool) {
743	pr_warn_ratelimited("Failed to allocate new pool");
744	return;
745	}
746
747	add_mem_pool(mem, pool);
748	}
749
750	/**
751	* swiotlb_dyn_free() - RCU callback to free a memory pool
752	* @rcu: RCU head in the corresponding struct io_tlb_pool.
753	*/
754	static void swiotlb_dyn_free(struct rcu_head *rcu)
755	{
756	struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
757	size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
758	size_t tlb_size = pool->end - pool->start;
759
760	free_pages(addr: (unsigned long)pool->slots, order: get_order(size: slots_size));
761	swiotlb_free_tlb(vaddr: pool->vaddr, bytes: tlb_size);
762	kfree(objp: pool);
763	}
764
765	/**
766	* swiotlb_find_pool() - find the IO TLB pool for a physical address
767	* @dev: Device which has mapped the DMA buffer.
768	* @paddr: Physical address within the DMA buffer.
769	*
770	* Find the IO TLB memory pool descriptor which contains the given physical
771	* address, if any.
772	*
773	* Return: Memory pool which contains @paddr, or %NULL if none.
774	*/
775	struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
776	{
777	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
778	struct io_tlb_pool *pool;
779
780	rcu_read_lock();
781	list_for_each_entry_rcu(pool, &mem->pools, node) {
782	if (paddr >= pool->start && paddr < pool->end)
783	goto out;
784	}
785
786	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
787	if (paddr >= pool->start && paddr < pool->end)
788	goto out;
789	}
790	pool = NULL;
791	out:
792	rcu_read_unlock();
793	return pool;
794	}
795
796	/**
797	* swiotlb_del_pool() - remove an IO TLB pool from a device
798	* @dev: Owning device.
799	* @pool: Memory pool to be removed.
800	*/
801	static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
802	{
803	unsigned long flags;
804
805	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
806	list_del_rcu(entry: &pool->node);
807	spin_unlock_irqrestore(lock: &dev->dma_io_tlb_lock, flags);
808
809	call_rcu(head: &pool->rcu, func: swiotlb_dyn_free);
810	}
811
812	#endif /* CONFIG_SWIOTLB_DYNAMIC */
813
814	/**
815	* swiotlb_dev_init() - initialize swiotlb fields in &struct device
816	* @dev: Device to be initialized.
817	*/
818	void swiotlb_dev_init(struct device *dev)
819	{
820	dev->dma_io_tlb_mem = &io_tlb_default_mem;
821	#ifdef CONFIG_SWIOTLB_DYNAMIC
822	INIT_LIST_HEAD(list: &dev->dma_io_tlb_pools);
823	spin_lock_init(&dev->dma_io_tlb_lock);
824	dev->dma_uses_io_tlb = false;
825	#endif
826	}
827
828	/**
829	* swiotlb_align_offset() - Get required offset into an IO TLB allocation.
830	* @dev: Owning device.
831	* @align_mask: Allocation alignment mask.
832	* @addr: DMA address.
833	*
834	* Return the minimum offset from the start of an IO TLB allocation which is
835	* required for a given buffer address and allocation alignment to keep the
836	* device happy.
837	*
838	* First, the address bits covered by min_align_mask must be identical in the
839	* original address and the bounce buffer address. High bits are preserved by
840	* choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
841	* padding bytes before the bounce buffer.
842	*
843	* Second, @align_mask specifies which bits of the first allocated slot must
844	* be zero. This may require allocating additional padding slots, and then the
845	* offset (in bytes) from the first such padding slot is returned.
846	*/
847	static unsigned int swiotlb_align_offset(struct device *dev,
848	unsigned int align_mask, u64 addr)
849	{
850	return addr & dma_get_min_align_mask(dev) &
851	(align_mask | (IO_TLB_SIZE - 1));
852	}
853
854	/*
855	* Bounce: copy the swiotlb buffer from or back to the original dma location
856	*/
857	static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
858	enum dma_data_direction dir)
859	{
860	struct io_tlb_pool *mem = swiotlb_find_pool(dev, paddr: tlb_addr);
861	int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
862	phys_addr_t orig_addr = mem->slots[index].orig_addr;
863	size_t alloc_size = mem->slots[index].alloc_size;
864	unsigned long pfn = PFN_DOWN(orig_addr);
865	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
866	int tlb_offset;
867
868	if (orig_addr == INVALID_PHYS_ADDR)
869	return;
870
871	/*
872	* It's valid for tlb_offset to be negative. This can happen when the
873	* "offset" returned by swiotlb_align_offset() is non-zero, and the
874	* tlb_addr is pointing within the first "offset" bytes of the second
875	* or subsequent slots of the allocated swiotlb area. While it's not
876	* valid for tlb_addr to be pointing within the first "offset" bytes
877	* of the first slot, there's no way to check for such an error since
878	* this function can't distinguish the first slot from the second and
879	* subsequent slots.
880	*/
881	tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
882	swiotlb_align_offset(dev, align_mask: 0, addr: orig_addr);
883
884	orig_addr += tlb_offset;
885	alloc_size -= tlb_offset;
886
887	if (size > alloc_size) {
888	dev_WARN_ONCE(dev, 1,
889	"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
890	alloc_size, size);
891	size = alloc_size;
892	}
893
894	if (PageHighMem(pfn_to_page(pfn))) {
895	unsigned int offset = orig_addr & ~PAGE_MASK;
896	struct page *page;
897	unsigned int sz = 0;
898	unsigned long flags;
899
900	while (size) {
901	sz = min_t(size_t, PAGE_SIZE - offset, size);
902
903	local_irq_save(flags);
904	page = pfn_to_page(pfn);
905	if (dir == DMA_TO_DEVICE)
906	memcpy_from_page(to: vaddr, page, offset, len: sz);
907	else
908	memcpy_to_page(page, offset, from: vaddr, len: sz);
909	local_irq_restore(flags);
910
911	size -= sz;
912	pfn++;
913	vaddr += sz;
914	offset = 0;
915	}
916	} else if (dir == DMA_TO_DEVICE) {
917	memcpy(vaddr, phys_to_virt(orig_addr), size);
918	} else {
919	memcpy(phys_to_virt(orig_addr), vaddr, size);
920	}
921	}
922
923	static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
924	{
925	return start + (idx << IO_TLB_SHIFT);
926	}
927
928	/*
929	* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
930	*/
931	static inline unsigned long get_max_slots(unsigned long boundary_mask)
932	{
933	return (boundary_mask >> IO_TLB_SHIFT) + 1;
934	}
935
936	static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
937	{
938	if (index >= mem->area_nslabs)
939	return 0;
940	return index;
941	}
942
943	/*
944	* Track the total used slots with a global atomic value in order to have
945	* correct information to determine the high water mark. The mem_used()
946	* function gives imprecise results because there's no locking across
947	* multiple areas.
948	*/
949	#ifdef CONFIG_DEBUG_FS
950	static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
951	{
952	unsigned long old_hiwater, new_used;
953
954	new_used = atomic_long_add_return(i: nslots, v: &mem->total_used);
955	old_hiwater = atomic_long_read(v: &mem->used_hiwater);
956	do {
957	if (new_used <= old_hiwater)
958	break;
959	} while (!atomic_long_try_cmpxchg(v: &mem->used_hiwater,
960	old: &old_hiwater, new: new_used));
961	}
962
963	static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
964	{
965	atomic_long_sub(i: nslots, v: &mem->total_used);
966	}
967
968	#else /* !CONFIG_DEBUG_FS */
969	static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
970	{
971	}
972	static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
973	{
974	}
975	#endif /* CONFIG_DEBUG_FS */
976
977	#ifdef CONFIG_SWIOTLB_DYNAMIC
978	#ifdef CONFIG_DEBUG_FS
979	static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
980	{
981	atomic_long_add(i: nslots, v: &mem->transient_nslabs);
982	}
983
984	static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
985	{
986	atomic_long_sub(i: nslots, v: &mem->transient_nslabs);
987	}
988
989	#else /* !CONFIG_DEBUG_FS */
990	static void inc_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
991	{
992	}
993	static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
994	{
995	}
996	#endif /* CONFIG_DEBUG_FS */
997	#endif /* CONFIG_SWIOTLB_DYNAMIC */
998
999	/**
1000	* swiotlb_search_pool_area() - search one memory area in one pool
1001	* @dev: Device which maps the buffer.
1002	* @pool: Memory pool to be searched.
1003	* @area_index: Index of the IO TLB memory area to be searched.
1004	* @orig_addr: Original (non-bounced) IO buffer address.
1005	* @alloc_size: Total requested size of the bounce buffer,
1006	* including initial alignment padding.
1007	* @alloc_align_mask: Required alignment of the allocated buffer.
1008	*
1009	* Find a suitable sequence of IO TLB entries for the request and allocate
1010	* a buffer from the given IO TLB memory area.
1011	* This function takes care of locking.
1012	*
1013	* Return: Index of the first allocated slot, or -1 on error.
1014	*/
1015	static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
1016	int area_index, phys_addr_t orig_addr, size_t alloc_size,
1017	unsigned int alloc_align_mask)
1018	{
1019	struct io_tlb_area *area = pool->areas + area_index;
1020	unsigned long boundary_mask = dma_get_seg_boundary(dev);
1021	dma_addr_t tbl_dma_addr =
1022	phys_to_dma_unencrypted(dev, paddr: pool->start) & boundary_mask;
1023	unsigned long max_slots = get_max_slots(boundary_mask);
1024	unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
1025	unsigned int nslots = nr_slots(val: alloc_size), stride;
1026	unsigned int offset = swiotlb_align_offset(dev, align_mask: 0, addr: orig_addr);
1027	unsigned int index, slots_checked, count = 0, i;
1028	unsigned long flags;
1029	unsigned int slot_base;
1030	unsigned int slot_index;
1031
1032	BUG_ON(!nslots);
1033	BUG_ON(area_index >= pool->nareas);
1034
1035	/*
1036	* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
1037	* page-aligned in the absence of any other alignment requirements.
1038	* 'alloc_align_mask' was later introduced to specify the alignment
1039	* explicitly, however this is passed as zero for streaming mappings
1040	* and so we preserve the old behaviour there in case any drivers are
1041	* relying on it.
1042	*/
1043	if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
1044	alloc_align_mask = PAGE_SIZE - 1;
1045
1046	/*
1047	* Ensure that the allocation is at least slot-aligned and update
1048	* 'iotlb_align_mask' to ignore bits that will be preserved when
1049	* offsetting into the allocation.
1050	*/
1051	alloc_align_mask |= (IO_TLB_SIZE - 1);
1052	iotlb_align_mask &= ~alloc_align_mask;
1053
1054	/*
1055	* For mappings with an alignment requirement don't bother looping to
1056	* unaligned slots once we found an aligned one.
1057	*/
1058	stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
1059
1060	spin_lock_irqsave(&area->lock, flags);
1061	if (unlikely(nslots > pool->area_nslabs - area->used))
1062	goto not_found;
1063
1064	slot_base = area_index * pool->area_nslabs;
1065	index = area->index;
1066
1067	for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1068	phys_addr_t tlb_addr;
1069
1070	slot_index = slot_base + index;
1071	tlb_addr = slot_addr(start: tbl_dma_addr, idx: slot_index);
1072
1073	if ((tlb_addr & alloc_align_mask) ||
1074	(orig_addr && (tlb_addr & iotlb_align_mask) !=
1075	(orig_addr & iotlb_align_mask))) {
1076	index = wrap_area_index(mem: pool, index: index + 1);
1077	slots_checked++;
1078	continue;
1079	}
1080
1081	if (!iommu_is_span_boundary(index: slot_index, nr: nslots,
1082	shift: nr_slots(val: tbl_dma_addr),
1083	boundary_size: max_slots)) {
1084	if (pool->slots[slot_index].list >= nslots)
1085	goto found;
1086	}
1087	index = wrap_area_index(mem: pool, index: index + stride);
1088	slots_checked += stride;
1089	}
1090
1091	not_found:
1092	spin_unlock_irqrestore(lock: &area->lock, flags);
1093	return -1;
1094
1095	found:
1096	/*
1097	* If we find a slot that indicates we have 'nslots' number of
1098	* contiguous buffers, we allocate the buffers from that slot onwards
1099	* and set the list of free entries to '0' indicating unavailable.
1100	*/
1101	for (i = slot_index; i < slot_index + nslots; i++) {
1102	pool->slots[i].list = 0;
1103	pool->slots[i].alloc_size = alloc_size - (offset +
1104	((i - slot_index) << IO_TLB_SHIFT));
1105	}
1106	for (i = slot_index - 1;
1107	io_tlb_offset(val: i) != IO_TLB_SEGSIZE - 1 &&
1108	pool->slots[i].list; i--)
1109	pool->slots[i].list = ++count;
1110
1111	/*
1112	* Update the indices to avoid searching in the next round.
1113	*/
1114	area->index = wrap_area_index(mem: pool, index: index + nslots);
1115	area->used += nslots;
1116	spin_unlock_irqrestore(lock: &area->lock, flags);
1117
1118	inc_used_and_hiwater(mem: dev->dma_io_tlb_mem, nslots);
1119	return slot_index;
1120	}
1121
1122	#ifdef CONFIG_SWIOTLB_DYNAMIC
1123
1124	/**
1125	* swiotlb_search_area() - search one memory area in all pools
1126	* @dev: Device which maps the buffer.
1127	* @start_cpu: Start CPU number.
1128	* @cpu_offset: Offset from @start_cpu.
1129	* @orig_addr: Original (non-bounced) IO buffer address.
1130	* @alloc_size: Total requested size of the bounce buffer,
1131	* including initial alignment padding.
1132	* @alloc_align_mask: Required alignment of the allocated buffer.
1133	* @retpool: Used memory pool, updated on return.
1134	*
1135	* Search one memory area in all pools for a sequence of slots that match the
1136	* allocation constraints.
1137	*
1138	* Return: Index of the first allocated slot, or -1 on error.
1139	*/
1140	static int swiotlb_search_area(struct device *dev, int start_cpu,
1141	int cpu_offset, phys_addr_t orig_addr, size_t alloc_size,
1142	unsigned int alloc_align_mask, struct io_tlb_pool **retpool)
1143	{
1144	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1145	struct io_tlb_pool *pool;
1146	int area_index;
1147	int index = -1;
1148
1149	rcu_read_lock();
1150	list_for_each_entry_rcu(pool, &mem->pools, node) {
1151	if (cpu_offset >= pool->nareas)
1152	continue;
1153	area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
1154	index = swiotlb_search_pool_area(dev, pool, area_index,
1155	orig_addr, alloc_size,
1156	alloc_align_mask);
1157	if (index >= 0) {
1158	*retpool = pool;
1159	break;
1160	}
1161	}
1162	rcu_read_unlock();
1163	return index;
1164	}
1165
1166	/**
1167	* swiotlb_find_slots() - search for slots in the whole swiotlb
1168	* @dev: Device which maps the buffer.
1169	* @orig_addr: Original (non-bounced) IO buffer address.
1170	* @alloc_size: Total requested size of the bounce buffer,
1171	* including initial alignment padding.
1172	* @alloc_align_mask: Required alignment of the allocated buffer.
1173	* @retpool: Used memory pool, updated on return.
1174	*
1175	* Search through the whole software IO TLB to find a sequence of slots that
1176	* match the allocation constraints.
1177	*
1178	* Return: Index of the first allocated slot, or -1 on error.
1179	*/
1180	static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1181	size_t alloc_size, unsigned int alloc_align_mask,
1182	struct io_tlb_pool **retpool)
1183	{
1184	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1185	struct io_tlb_pool *pool;
1186	unsigned long nslabs;
1187	unsigned long flags;
1188	u64 phys_limit;
1189	int cpu, i;
1190	int index;
1191
1192	if (alloc_size > IO_TLB_SEGSIZE * IO_TLB_SIZE)
1193	return -1;
1194
1195	cpu = raw_smp_processor_id();
1196	for (i = 0; i < default_nareas; ++i) {
1197	index = swiotlb_search_area(dev, start_cpu: cpu, cpu_offset: i, orig_addr, alloc_size,
1198	alloc_align_mask, retpool: &pool);
1199	if (index >= 0)
1200	goto found;
1201	}
1202
1203	if (!mem->can_grow)
1204	return -1;
1205
1206	schedule_work(work: &mem->dyn_alloc);
1207
1208	nslabs = nr_slots(val: alloc_size);
1209	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1210	pool = swiotlb_alloc_pool(dev, minslabs: nslabs, nslabs, nareas: 1, phys_limit,
1211	GFP_NOWAIT | __GFP_NOWARN);
1212	if (!pool)
1213	return -1;
1214
1215	index = swiotlb_search_pool_area(dev, pool, area_index: 0, orig_addr,
1216	alloc_size, alloc_align_mask);
1217	if (index < 0) {
1218	swiotlb_dyn_free(rcu: &pool->rcu);
1219	return -1;
1220	}
1221
1222	pool->transient = true;
1223	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1224	list_add_rcu(new: &pool->node, head: &dev->dma_io_tlb_pools);
1225	spin_unlock_irqrestore(lock: &dev->dma_io_tlb_lock, flags);
1226	inc_transient_used(mem, nslots: pool->nslabs);
1227
1228	found:
1229	WRITE_ONCE(dev->dma_uses_io_tlb, true);
1230
1231	/*
1232	* The general barrier orders reads and writes against a presumed store
1233	* of the SWIOTLB buffer address by a device driver (to a driver private
1234	* data structure). It serves two purposes.
1235	*
1236	* First, the store to dev->dma_uses_io_tlb must be ordered before the
1237	* presumed store. This guarantees that the returned buffer address
1238	* cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1239	*
1240	* Second, the load from mem->pools must be ordered before the same
1241	* presumed store. This guarantees that the returned buffer address
1242	* cannot be observed by another CPU before an update of the RCU list
1243	* that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1244	* atomicity).
1245	*
1246	* See also the comment in is_swiotlb_buffer().
1247	*/
1248	smp_mb();
1249
1250	*retpool = pool;
1251	return index;
1252	}
1253
1254	#else /* !CONFIG_SWIOTLB_DYNAMIC */
1255
1256	static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1257	size_t alloc_size, unsigned int alloc_align_mask,
1258	struct io_tlb_pool **retpool)
1259	{
1260	struct io_tlb_pool *pool;
1261	int start, i;
1262	int index;
1263
1264	*retpool = pool = &dev->dma_io_tlb_mem->defpool;
1265	i = start = raw_smp_processor_id() & (pool->nareas - 1);
1266	do {
1267	index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
1268	alloc_size, alloc_align_mask);
1269	if (index >= 0)
1270	return index;
1271	if (++i >= pool->nareas)
1272	i = 0;
1273	} while (i != start);
1274	return -1;
1275	}
1276
1277	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1278
1279	#ifdef CONFIG_DEBUG_FS
1280
1281	/**
1282	* mem_used() - get number of used slots in an allocator
1283	* @mem: Software IO TLB allocator.
1284	*
1285	* The result is accurate in this version of the function, because an atomic
1286	* counter is available if CONFIG_DEBUG_FS is set.
1287	*
1288	* Return: Number of used slots.
1289	*/
1290	static unsigned long mem_used(struct io_tlb_mem *mem)
1291	{
1292	return atomic_long_read(v: &mem->total_used);
1293	}
1294
1295	#else /* !CONFIG_DEBUG_FS */
1296
1297	/**
1298	* mem_pool_used() - get number of used slots in a memory pool
1299	* @pool: Software IO TLB memory pool.
1300	*
1301	* The result is not accurate, see mem_used().
1302	*
1303	* Return: Approximate number of used slots.
1304	*/
1305	static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1306	{
1307	int i;
1308	unsigned long used = 0;
1309
1310	for (i = 0; i < pool->nareas; i++)
1311	used += pool->areas[i].used;
1312	return used;
1313	}
1314
1315	/**
1316	* mem_used() - get number of used slots in an allocator
1317	* @mem: Software IO TLB allocator.
1318	*
1319	* The result is not accurate, because there is no locking of individual
1320	* areas.
1321	*
1322	* Return: Approximate number of used slots.
1323	*/
1324	static unsigned long mem_used(struct io_tlb_mem *mem)
1325	{
1326	#ifdef CONFIG_SWIOTLB_DYNAMIC
1327	struct io_tlb_pool *pool;
1328	unsigned long used = 0;
1329
1330	rcu_read_lock();
1331	list_for_each_entry_rcu(pool, &mem->pools, node)
1332	used += mem_pool_used(pool);
1333	rcu_read_unlock();
1334
1335	return used;
1336	#else
1337	return mem_pool_used(&mem->defpool);
1338	#endif
1339	}
1340
1341	#endif /* CONFIG_DEBUG_FS */
1342
1343	phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1344	size_t mapping_size, size_t alloc_size,
1345	unsigned int alloc_align_mask, enum dma_data_direction dir,
1346	unsigned long attrs)
1347	{
1348	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1349	unsigned int offset;
1350	struct io_tlb_pool *pool;
1351	unsigned int i;
1352	int index;
1353	phys_addr_t tlb_addr;
1354	unsigned short pad_slots;
1355
1356	if (!mem || !mem->nslabs) {
1357	dev_warn_ratelimited(dev,
1358	"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1359	return (phys_addr_t)DMA_MAPPING_ERROR;
1360	}
1361
1362	if (cc_platform_has(attr: CC_ATTR_MEM_ENCRYPT))
1363	pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1364
1365	if (mapping_size > alloc_size) {
1366	dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
1367	mapping_size, alloc_size);
1368	return (phys_addr_t)DMA_MAPPING_ERROR;
1369	}
1370
1371	offset = swiotlb_align_offset(dev, align_mask: alloc_align_mask, addr: orig_addr);
1372	index = swiotlb_find_slots(dev, orig_addr,
1373	alloc_size: alloc_size + offset, alloc_align_mask, retpool: &pool);
1374	if (index == -1) {
1375	if (!(attrs & DMA_ATTR_NO_WARN))
1376	dev_warn_ratelimited(dev,
1377	"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1378	alloc_size, mem->nslabs, mem_used(mem));
1379	return (phys_addr_t)DMA_MAPPING_ERROR;
1380	}
1381
1382	/*
1383	* Save away the mapping from the original address to the DMA address.
1384	* This is needed when we sync the memory. Then we sync the buffer if
1385	* needed.
1386	*/
1387	pad_slots = offset >> IO_TLB_SHIFT;
1388	offset &= (IO_TLB_SIZE - 1);
1389	index += pad_slots;
1390	pool->slots[index].pad_slots = pad_slots;
1391	for (i = 0; i < nr_slots(val: alloc_size + offset); i++)
1392	pool->slots[index + i].orig_addr = slot_addr(start: orig_addr, idx: i);
1393	tlb_addr = slot_addr(start: pool->start, idx: index) + offset;
1394	/*
1395	* When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1396	* the original buffer to the TLB buffer before initiating DMA in order
1397	* to preserve the original's data if the device does a partial write,
1398	* i.e. if the device doesn't overwrite the entire buffer. Preserving
1399	* the original data, even if it's garbage, is necessary to match
1400	* hardware behavior. Use of swiotlb is supposed to be transparent,
1401	* i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1402	*/
1403	swiotlb_bounce(dev, tlb_addr, size: mapping_size, dir: DMA_TO_DEVICE);
1404	return tlb_addr;
1405	}
1406
1407	static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
1408	{
1409	struct io_tlb_pool *mem = swiotlb_find_pool(dev, paddr: tlb_addr);
1410	unsigned long flags;
1411	unsigned int offset = swiotlb_align_offset(dev, align_mask: 0, addr: tlb_addr);
1412	int index, nslots, aindex;
1413	struct io_tlb_area *area;
1414	int count, i;
1415
1416	index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1417	index -= mem->slots[index].pad_slots;
1418	nslots = nr_slots(val: mem->slots[index].alloc_size + offset);
1419	aindex = index / mem->area_nslabs;
1420	area = &mem->areas[aindex];
1421
1422	/*
1423	* Return the buffer to the free list by setting the corresponding
1424	* entries to indicate the number of contiguous entries available.
1425	* While returning the entries to the free list, we merge the entries
1426	* with slots below and above the pool being returned.
1427	*/
1428	BUG_ON(aindex >= mem->nareas);
1429
1430	spin_lock_irqsave(&area->lock, flags);
1431	if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1432	count = mem->slots[index + nslots].list;
1433	else
1434	count = 0;
1435
1436	/*
1437	* Step 1: return the slots to the free list, merging the slots with
1438	* superceeding slots
1439	*/
1440	for (i = index + nslots - 1; i >= index; i--) {
1441	mem->slots[i].list = ++count;
1442	mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1443	mem->slots[i].alloc_size = 0;
1444	mem->slots[i].pad_slots = 0;
1445	}
1446
1447	/*
1448	* Step 2: merge the returned slots with the preceding slots, if
1449	* available (non zero)
1450	*/
1451	for (i = index - 1;
1452	io_tlb_offset(val: i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1453	i--)
1454	mem->slots[i].list = ++count;
1455	area->used -= nslots;
1456	spin_unlock_irqrestore(lock: &area->lock, flags);
1457
1458	dec_used(mem: dev->dma_io_tlb_mem, nslots);
1459	}
1460
1461	#ifdef CONFIG_SWIOTLB_DYNAMIC
1462
1463	/**
1464	* swiotlb_del_transient() - delete a transient memory pool
1465	* @dev: Device which mapped the buffer.
1466	* @tlb_addr: Physical address within a bounce buffer.
1467	*
1468	* Check whether the address belongs to a transient SWIOTLB memory pool.
1469	* If yes, then delete the pool.
1470	*
1471	* Return: %true if @tlb_addr belonged to a transient pool that was released.
1472	*/
1473	static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr)
1474	{
1475	struct io_tlb_pool *pool;
1476
1477	pool = swiotlb_find_pool(dev, paddr: tlb_addr);
1478	if (!pool->transient)
1479	return false;
1480
1481	dec_used(mem: dev->dma_io_tlb_mem, nslots: pool->nslabs);
1482	swiotlb_del_pool(dev, pool);
1483	dec_transient_used(mem: dev->dma_io_tlb_mem, nslots: pool->nslabs);
1484	return true;
1485	}
1486
1487	#else /* !CONFIG_SWIOTLB_DYNAMIC */
1488
1489	static inline bool swiotlb_del_transient(struct device *dev,
1490	phys_addr_t tlb_addr)
1491	{
1492	return false;
1493	}
1494
1495	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1496
1497	/*
1498	* tlb_addr is the physical address of the bounce buffer to unmap.
1499	*/
1500	void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1501	size_t mapping_size, enum dma_data_direction dir,
1502	unsigned long attrs)
1503	{
1504	/*
1505	* First, sync the memory before unmapping the entry
1506	*/
1507	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1508	(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1509	swiotlb_bounce(dev, tlb_addr, size: mapping_size, dir: DMA_FROM_DEVICE);
1510
1511	if (swiotlb_del_transient(dev, tlb_addr))
1512	return;
1513	swiotlb_release_slots(dev, tlb_addr);
1514	}
1515
1516	void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1517	size_t size, enum dma_data_direction dir)
1518	{
1519	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1520	swiotlb_bounce(dev, tlb_addr, size, dir: DMA_TO_DEVICE);
1521	else
1522	BUG_ON(dir != DMA_FROM_DEVICE);
1523	}
1524
1525	void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1526	size_t size, enum dma_data_direction dir)
1527	{
1528	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1529	swiotlb_bounce(dev, tlb_addr, size, dir: DMA_FROM_DEVICE);
1530	else
1531	BUG_ON(dir != DMA_TO_DEVICE);
1532	}
1533
1534	/*
1535	* Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1536	* to the device copy the data into it as well.
1537	*/
1538	dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1539	enum dma_data_direction dir, unsigned long attrs)
1540	{
1541	phys_addr_t swiotlb_addr;
1542	dma_addr_t dma_addr;
1543
1544	trace_swiotlb_bounced(dev, dev_addr: phys_to_dma(dev, paddr), size);
1545
1546	swiotlb_addr = swiotlb_tbl_map_single(dev, orig_addr: paddr, mapping_size: size, alloc_size: size, alloc_align_mask: 0, dir,
1547	attrs);
1548	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1549	return DMA_MAPPING_ERROR;
1550
1551	/* Ensure that the address returned is DMA'ble */
1552	dma_addr = phys_to_dma_unencrypted(dev, paddr: swiotlb_addr);
1553	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1554	swiotlb_tbl_unmap_single(dev, tlb_addr: swiotlb_addr, mapping_size: size, dir,
1555	attrs: attrs | DMA_ATTR_SKIP_CPU_SYNC);
1556	dev_WARN_ONCE(dev, 1,
1557	"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1558	&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1559	return DMA_MAPPING_ERROR;
1560	}
1561
1562	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1563	arch_sync_dma_for_device(paddr: swiotlb_addr, size, dir);
1564	return dma_addr;
1565	}
1566
1567	size_t swiotlb_max_mapping_size(struct device *dev)
1568	{
1569	int min_align_mask = dma_get_min_align_mask(dev);
1570	int min_align = 0;
1571
1572	/*
1573	* swiotlb_find_slots() skips slots according to
1574	* min align mask. This affects max mapping size.
1575	* Take it into acount here.
1576	*/
1577	if (min_align_mask)
1578	min_align = roundup(min_align_mask, IO_TLB_SIZE);
1579
1580	return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1581	}
1582
1583	/**
1584	* is_swiotlb_allocated() - check if the default software IO TLB is initialized
1585	*/
1586	bool is_swiotlb_allocated(void)
1587	{
1588	return io_tlb_default_mem.nslabs;
1589	}
1590
1591	bool is_swiotlb_active(struct device *dev)
1592	{
1593	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1594
1595	return mem && mem->nslabs;
1596	}
1597
1598	/**
1599	* default_swiotlb_base() - get the base address of the default SWIOTLB
1600	*
1601	* Get the lowest physical address used by the default software IO TLB pool.
1602	*/
1603	phys_addr_t default_swiotlb_base(void)
1604	{
1605	#ifdef CONFIG_SWIOTLB_DYNAMIC
1606	io_tlb_default_mem.can_grow = false;
1607	#endif
1608	return io_tlb_default_mem.defpool.start;
1609	}
1610
1611	/**
1612	* default_swiotlb_limit() - get the address limit of the default SWIOTLB
1613	*
1614	* Get the highest physical address used by the default software IO TLB pool.
1615	*/
1616	phys_addr_t default_swiotlb_limit(void)
1617	{
1618	#ifdef CONFIG_SWIOTLB_DYNAMIC
1619	return io_tlb_default_mem.phys_limit;
1620	#else
1621	return io_tlb_default_mem.defpool.end - 1;
1622	#endif
1623	}
1624
1625	#ifdef CONFIG_DEBUG_FS
1626	#ifdef CONFIG_SWIOTLB_DYNAMIC
1627	static unsigned long mem_transient_used(struct io_tlb_mem *mem)
1628	{
1629	return atomic_long_read(v: &mem->transient_nslabs);
1630	}
1631
1632	static int io_tlb_transient_used_get(void *data, u64 *val)
1633	{
1634	struct io_tlb_mem *mem = data;
1635
1636	*val = mem_transient_used(mem);
1637	return 0;
1638	}
1639
1640	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_transient_used, io_tlb_transient_used_get,
1641	NULL, "%llu\n");
1642	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1643
1644	static int io_tlb_used_get(void *data, u64 *val)
1645	{
1646	struct io_tlb_mem *mem = data;
1647
1648	*val = mem_used(mem);
1649	return 0;
1650	}
1651
1652	static int io_tlb_hiwater_get(void *data, u64 *val)
1653	{
1654	struct io_tlb_mem *mem = data;
1655
1656	*val = atomic_long_read(v: &mem->used_hiwater);
1657	return 0;
1658	}
1659
1660	static int io_tlb_hiwater_set(void *data, u64 val)
1661	{
1662	struct io_tlb_mem *mem = data;
1663
1664	/* Only allow setting to zero */
1665	if (val != 0)
1666	return -EINVAL;
1667
1668	atomic_long_set(v: &mem->used_hiwater, i: val);
1669	return 0;
1670	}
1671
1672	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1673	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1674	io_tlb_hiwater_set, "%llu\n");
1675
1676	static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1677	const char *dirname)
1678	{
1679	mem->debugfs = debugfs_create_dir(name: dirname, parent: io_tlb_default_mem.debugfs);
1680	if (!mem->nslabs)
1681	return;
1682
1683	debugfs_create_ulong(name: "io_tlb_nslabs", mode: 0400, parent: mem->debugfs, value: &mem->nslabs);
1684	debugfs_create_file(name: "io_tlb_used", mode: 0400, parent: mem->debugfs, data: mem,
1685	fops: &fops_io_tlb_used);
1686	debugfs_create_file(name: "io_tlb_used_hiwater", mode: 0600, parent: mem->debugfs, data: mem,
1687	fops: &fops_io_tlb_hiwater);
1688	#ifdef CONFIG_SWIOTLB_DYNAMIC
1689	debugfs_create_file(name: "io_tlb_transient_nslabs", mode: 0400, parent: mem->debugfs,
1690	data: mem, fops: &fops_io_tlb_transient_used);
1691	#endif
1692	}
1693
1694	static int __init swiotlb_create_default_debugfs(void)
1695	{
1696	swiotlb_create_debugfs_files(mem: &io_tlb_default_mem, dirname: "swiotlb");
1697	return 0;
1698	}
1699
1700	late_initcall(swiotlb_create_default_debugfs);
1701
1702	#else /* !CONFIG_DEBUG_FS */
1703
1704	static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1705	const char *dirname)
1706	{
1707	}
1708
1709	#endif /* CONFIG_DEBUG_FS */
1710
1711	#ifdef CONFIG_DMA_RESTRICTED_POOL
1712
1713	struct page *swiotlb_alloc(struct device *dev, size_t size)
1714	{
1715	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1716	struct io_tlb_pool *pool;
1717	phys_addr_t tlb_addr;
1718	unsigned int align;
1719	int index;
1720
1721	if (!mem)
1722	return NULL;
1723
1724	align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
1725	index = swiotlb_find_slots(dev, orig_addr: 0, alloc_size: size, alloc_align_mask: align, retpool: &pool);
1726	if (index == -1)
1727	return NULL;
1728
1729	tlb_addr = slot_addr(start: pool->start, idx: index);
1730	if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
1731	dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
1732	&tlb_addr);
1733	swiotlb_release_slots(dev, tlb_addr);
1734	return NULL;
1735	}
1736
1737	return pfn_to_page(PFN_DOWN(tlb_addr));
1738	}
1739
1740	bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1741	{
1742	phys_addr_t tlb_addr = page_to_phys(page);
1743
1744	if (!is_swiotlb_buffer(dev, paddr: tlb_addr))
1745	return false;
1746
1747	swiotlb_release_slots(dev, tlb_addr);
1748
1749	return true;
1750	}
1751
1752	static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1753	struct device *dev)
1754	{
1755	struct io_tlb_mem *mem = rmem->priv;
1756	unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1757
1758	/* Set Per-device io tlb area to one */
1759	unsigned int nareas = 1;
1760
1761	if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1762	dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1763	return -EINVAL;
1764	}
1765
1766	/*
1767	* Since multiple devices can share the same pool, the private data,
1768	* io_tlb_mem struct, will be initialized by the first device attached
1769	* to it.
1770	*/
1771	if (!mem) {
1772	struct io_tlb_pool *pool;
1773
1774	mem = kzalloc(size: sizeof(*mem), GFP_KERNEL);
1775	if (!mem)
1776	return -ENOMEM;
1777	pool = &mem->defpool;
1778
1779	pool->slots = kcalloc(n: nslabs, size: sizeof(*pool->slots), GFP_KERNEL);
1780	if (!pool->slots) {
1781	kfree(objp: mem);
1782	return -ENOMEM;
1783	}
1784
1785	pool->areas = kcalloc(n: nareas, size: sizeof(*pool->areas),
1786	GFP_KERNEL);
1787	if (!pool->areas) {
1788	kfree(objp: pool->slots);
1789	kfree(objp: mem);
1790	return -ENOMEM;
1791	}
1792
1793	set_memory_decrypted(addr: (unsigned long)phys_to_virt(address: rmem->base),
1794	numpages: rmem->size >> PAGE_SHIFT);
1795	swiotlb_init_io_tlb_pool(mem: pool, start: rmem->base, nslabs,
1796	late_alloc: false, nareas);
1797	mem->force_bounce = true;
1798	mem->for_alloc = true;
1799	#ifdef CONFIG_SWIOTLB_DYNAMIC
1800	spin_lock_init(&mem->lock);
1801	#endif
1802	add_mem_pool(mem, pool);
1803
1804	rmem->priv = mem;
1805
1806	swiotlb_create_debugfs_files(mem, dirname: rmem->name);
1807	}
1808
1809	dev->dma_io_tlb_mem = mem;
1810
1811	return 0;
1812	}
1813
1814	static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1815	struct device *dev)
1816	{
1817	dev->dma_io_tlb_mem = &io_tlb_default_mem;
1818	}
1819
1820	static const struct reserved_mem_ops rmem_swiotlb_ops = {
1821	.device_init = rmem_swiotlb_device_init,
1822	.device_release = rmem_swiotlb_device_release,
1823	};
1824
1825	static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1826	{
1827	unsigned long node = rmem->fdt_node;
1828
1829	if (of_get_flat_dt_prop(node, name: "reusable", NULL) ||
1830	of_get_flat_dt_prop(node, name: "linux,cma-default", NULL) ||
1831	of_get_flat_dt_prop(node, name: "linux,dma-default", NULL) ||
1832	of_get_flat_dt_prop(node, name: "no-map", NULL))
1833	return -EINVAL;
1834
1835	rmem->ops = &rmem_swiotlb_ops;
1836	pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1837	&rmem->base, (unsigned long)rmem->size / SZ_1M);
1838	return 0;
1839	}
1840
1841	RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1842	#endif /* CONFIG_DMA_RESTRICTED_POOL */
1843