1 | // SPDX-License-Identifier: GPL-2.0 |
---|---|
2 | /* |
3 | * Copyright (C) 2018-2020 Christoph Hellwig. |
4 | * |
5 | * DMA operations that map physical memory directly without using an IOMMU. |
6 | */ |
7 | #include <linux/memblock.h> /* for max_pfn */ |
8 | #include <linux/export.h> |
9 | #include <linux/mm.h> |
10 | #include <linux/dma-map-ops.h> |
11 | #include <linux/scatterlist.h> |
12 | #include <linux/pfn.h> |
13 | #include <linux/vmalloc.h> |
14 | #include <linux/set_memory.h> |
15 | #include <linux/slab.h> |
16 | #include <linux/pci-p2pdma.h> |
17 | #include "direct.h" |
18 | |
19 | /* |
20 | * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use |
21 | * it for entirely different regions. In that case the arch code needs to |
22 | * override the variable below for dma-direct to work properly. |
23 | */ |
24 | u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24); |
25 | |
26 | static inline dma_addr_t phys_to_dma_direct(struct device *dev, |
27 | phys_addr_t phys) |
28 | { |
29 | if (force_dma_unencrypted(dev)) |
30 | return phys_to_dma_unencrypted(dev, paddr: phys); |
31 | return phys_to_dma(dev, paddr: phys); |
32 | } |
33 | |
34 | static inline struct page *dma_direct_to_page(struct device *dev, |
35 | dma_addr_t dma_addr) |
36 | { |
37 | return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr))); |
38 | } |
39 | |
40 | u64 dma_direct_get_required_mask(struct device *dev) |
41 | { |
42 | phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT; |
43 | u64 max_dma = phys_to_dma_direct(dev, phys); |
44 | |
45 | return (1ULL << (fls64(x: max_dma) - 1)) * 2 - 1; |
46 | } |
47 | |
48 | static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit) |
49 | { |
50 | u64 dma_limit = min_not_zero( |
51 | dev->coherent_dma_mask, |
52 | dev->bus_dma_limit); |
53 | |
54 | /* |
55 | * Optimistically try the zone that the physical address mask falls |
56 | * into first. If that returns memory that isn't actually addressable |
57 | * we will fallback to the next lower zone and try again. |
58 | * |
59 | * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding |
60 | * zones. |
61 | */ |
62 | *phys_limit = dma_to_phys(dev, dma_addr: dma_limit); |
63 | if (*phys_limit <= zone_dma_limit) |
64 | return GFP_DMA; |
65 | if (*phys_limit <= DMA_BIT_MASK(32)) |
66 | return GFP_DMA32; |
67 | return 0; |
68 | } |
69 | |
70 | bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) |
71 | { |
72 | dma_addr_t dma_addr = phys_to_dma_direct(dev, phys); |
73 | |
74 | if (dma_addr == DMA_MAPPING_ERROR) |
75 | return false; |
76 | return dma_addr + size - 1 <= |
77 | min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); |
78 | } |
79 | |
80 | static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size) |
81 | { |
82 | if (!force_dma_unencrypted(dev)) |
83 | return 0; |
84 | return set_memory_decrypted(addr: (unsigned long)vaddr, PFN_UP(size)); |
85 | } |
86 | |
87 | static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size) |
88 | { |
89 | int ret; |
90 | |
91 | if (!force_dma_unencrypted(dev)) |
92 | return 0; |
93 | ret = set_memory_encrypted(addr: (unsigned long)vaddr, PFN_UP(size)); |
94 | if (ret) |
95 | pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); |
96 | return ret; |
97 | } |
98 | |
99 | static void __dma_direct_free_pages(struct device *dev, struct page *page, |
100 | size_t size) |
101 | { |
102 | if (swiotlb_free(dev, page, size)) |
103 | return; |
104 | dma_free_contiguous(dev, page, size); |
105 | } |
106 | |
107 | static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size) |
108 | { |
109 | struct page *page = swiotlb_alloc(dev, size); |
110 | |
111 | if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { |
112 | swiotlb_free(dev, page, size); |
113 | return NULL; |
114 | } |
115 | |
116 | return page; |
117 | } |
118 | |
119 | static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, |
120 | gfp_t gfp, bool allow_highmem) |
121 | { |
122 | int node = dev_to_node(dev); |
123 | struct page *page = NULL; |
124 | u64 phys_limit; |
125 | |
126 | WARN_ON_ONCE(!PAGE_ALIGNED(size)); |
127 | |
128 | if (is_swiotlb_for_alloc(dev)) |
129 | return dma_direct_alloc_swiotlb(dev, size); |
130 | |
131 | gfp |= dma_direct_optimal_gfp_mask(dev, phys_limit: &phys_limit); |
132 | page = dma_alloc_contiguous(dev, size, gfp); |
133 | if (page) { |
134 | if (!dma_coherent_ok(dev, page_to_phys(page), size) || |
135 | (!allow_highmem && PageHighMem(page))) { |
136 | dma_free_contiguous(dev, page, size); |
137 | page = NULL; |
138 | } |
139 | } |
140 | again: |
141 | if (!page) |
142 | page = alloc_pages_node(node, gfp, get_order(size)); |
143 | if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { |
144 | __free_pages(page, order: get_order(size)); |
145 | page = NULL; |
146 | |
147 | if (IS_ENABLED(CONFIG_ZONE_DMA32) && |
148 | phys_limit < DMA_BIT_MASK(64) && |
149 | !(gfp & (GFP_DMA32 | GFP_DMA))) { |
150 | gfp |= GFP_DMA32; |
151 | goto again; |
152 | } |
153 | |
154 | if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { |
155 | gfp = (gfp & ~GFP_DMA32) | GFP_DMA; |
156 | goto again; |
157 | } |
158 | } |
159 | |
160 | return page; |
161 | } |
162 | |
163 | /* |
164 | * Check if a potentially blocking operations needs to dip into the atomic |
165 | * pools for the given device/gfp. |
166 | */ |
167 | static bool dma_direct_use_pool(struct device *dev, gfp_t gfp) |
168 | { |
169 | return !gfpflags_allow_blocking(gfp_flags: gfp) && !is_swiotlb_for_alloc(dev); |
170 | } |
171 | |
172 | static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, |
173 | dma_addr_t *dma_handle, gfp_t gfp) |
174 | { |
175 | struct page *page; |
176 | u64 phys_limit; |
177 | void *ret; |
178 | |
179 | if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))) |
180 | return NULL; |
181 | |
182 | gfp |= dma_direct_optimal_gfp_mask(dev, phys_limit: &phys_limit); |
183 | page = dma_alloc_from_pool(dev, size, cpu_addr: &ret, flags: gfp, phys_addr_ok: dma_coherent_ok); |
184 | if (!page) |
185 | return NULL; |
186 | *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); |
187 | return ret; |
188 | } |
189 | |
190 | static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, |
191 | dma_addr_t *dma_handle, gfp_t gfp) |
192 | { |
193 | struct page *page; |
194 | |
195 | page = __dma_direct_alloc_pages(dev, size, gfp: gfp & ~__GFP_ZERO, allow_highmem: true); |
196 | if (!page) |
197 | return NULL; |
198 | |
199 | /* remove any dirty cache lines on the kernel alias */ |
200 | if (!PageHighMem(page)) |
201 | arch_dma_prep_coherent(page, size); |
202 | |
203 | /* return the page pointer as the opaque cookie */ |
204 | *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); |
205 | return page; |
206 | } |
207 | |
208 | void *dma_direct_alloc(struct device *dev, size_t size, |
209 | dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) |
210 | { |
211 | bool remap = false, set_uncached = false; |
212 | struct page *page; |
213 | void *ret; |
214 | |
215 | size = PAGE_ALIGN(size); |
216 | if (attrs & DMA_ATTR_NO_WARN) |
217 | gfp |= __GFP_NOWARN; |
218 | |
219 | if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && |
220 | !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) |
221 | return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); |
222 | |
223 | if (!dev_is_dma_coherent(dev)) { |
224 | if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && |
225 | !is_swiotlb_for_alloc(dev)) |
226 | return arch_dma_alloc(dev, size, dma_handle, gfp, |
227 | attrs); |
228 | |
229 | /* |
230 | * If there is a global pool, always allocate from it for |
231 | * non-coherent devices. |
232 | */ |
233 | if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL)) |
234 | return dma_alloc_from_global_coherent(dev, size, |
235 | dma_handle); |
236 | |
237 | /* |
238 | * Otherwise we require the architecture to either be able to |
239 | * mark arbitrary parts of the kernel direct mapping uncached, |
240 | * or remapped it uncached. |
241 | */ |
242 | set_uncached = IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED); |
243 | remap = IS_ENABLED(CONFIG_DMA_DIRECT_REMAP); |
244 | if (!set_uncached && !remap) { |
245 | pr_warn_once("coherent DMA allocations not supported on this platform.\n"); |
246 | return NULL; |
247 | } |
248 | } |
249 | |
250 | /* |
251 | * Remapping or decrypting memory may block, allocate the memory from |
252 | * the atomic pools instead if we aren't allowed block. |
253 | */ |
254 | if ((remap || force_dma_unencrypted(dev)) && |
255 | dma_direct_use_pool(dev, gfp)) |
256 | return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); |
257 | |
258 | /* we always manually zero the memory once we are done */ |
259 | page = __dma_direct_alloc_pages(dev, size, gfp: gfp & ~__GFP_ZERO, allow_highmem: true); |
260 | if (!page) |
261 | return NULL; |
262 | |
263 | /* |
264 | * dma_alloc_contiguous can return highmem pages depending on a |
265 | * combination the cma= arguments and per-arch setup. These need to be |
266 | * remapped to return a kernel virtual address. |
267 | */ |
268 | if (PageHighMem(page)) { |
269 | remap = true; |
270 | set_uncached = false; |
271 | } |
272 | |
273 | if (remap) { |
274 | pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); |
275 | |
276 | if (force_dma_unencrypted(dev)) |
277 | prot = pgprot_decrypted(prot); |
278 | |
279 | /* remove any dirty cache lines on the kernel alias */ |
280 | arch_dma_prep_coherent(page, size); |
281 | |
282 | /* create a coherent mapping */ |
283 | ret = dma_common_contiguous_remap(page, size, prot, |
284 | caller: __builtin_return_address(0)); |
285 | if (!ret) |
286 | goto out_free_pages; |
287 | } else { |
288 | ret = page_address(page); |
289 | if (dma_set_decrypted(dev, vaddr: ret, size)) |
290 | goto out_leak_pages; |
291 | } |
292 | |
293 | memset(ret, 0, size); |
294 | |
295 | if (set_uncached) { |
296 | arch_dma_prep_coherent(page, size); |
297 | ret = arch_dma_set_uncached(addr: ret, size); |
298 | if (IS_ERR(ptr: ret)) |
299 | goto out_encrypt_pages; |
300 | } |
301 | |
302 | *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); |
303 | return ret; |
304 | |
305 | out_encrypt_pages: |
306 | if (dma_set_encrypted(dev, page_address(page), size)) |
307 | return NULL; |
308 | out_free_pages: |
309 | __dma_direct_free_pages(dev, page, size); |
310 | return NULL; |
311 | out_leak_pages: |
312 | return NULL; |
313 | } |
314 | |
315 | void dma_direct_free(struct device *dev, size_t size, |
316 | void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) |
317 | { |
318 | unsigned int page_order = get_order(size); |
319 | |
320 | if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && |
321 | !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) { |
322 | /* cpu_addr is a struct page cookie, not a kernel address */ |
323 | dma_free_contiguous(dev, page: cpu_addr, size); |
324 | return; |
325 | } |
326 | |
327 | if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && |
328 | !dev_is_dma_coherent(dev) && |
329 | !is_swiotlb_for_alloc(dev)) { |
330 | arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); |
331 | return; |
332 | } |
333 | |
334 | if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) && |
335 | !dev_is_dma_coherent(dev)) { |
336 | if (!dma_release_from_global_coherent(order: page_order, vaddr: cpu_addr)) |
337 | WARN_ON_ONCE(1); |
338 | return; |
339 | } |
340 | |
341 | /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ |
342 | if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && |
343 | dma_free_from_pool(dev, start: cpu_addr, PAGE_ALIGN(size))) |
344 | return; |
345 | |
346 | if (is_vmalloc_addr(x: cpu_addr)) { |
347 | vunmap(addr: cpu_addr); |
348 | } else { |
349 | if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) |
350 | arch_dma_clear_uncached(addr: cpu_addr, size); |
351 | if (dma_set_encrypted(dev, vaddr: cpu_addr, size)) |
352 | return; |
353 | } |
354 | |
355 | __dma_direct_free_pages(dev, page: dma_direct_to_page(dev, dma_addr), size); |
356 | } |
357 | |
358 | struct page *dma_direct_alloc_pages(struct device *dev, size_t size, |
359 | dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) |
360 | { |
361 | struct page *page; |
362 | void *ret; |
363 | |
364 | if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) |
365 | return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); |
366 | |
367 | page = __dma_direct_alloc_pages(dev, size, gfp, allow_highmem: false); |
368 | if (!page) |
369 | return NULL; |
370 | |
371 | ret = page_address(page); |
372 | if (dma_set_decrypted(dev, vaddr: ret, size)) |
373 | goto out_leak_pages; |
374 | memset(ret, 0, size); |
375 | *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); |
376 | return page; |
377 | out_leak_pages: |
378 | return NULL; |
379 | } |
380 | |
381 | void dma_direct_free_pages(struct device *dev, size_t size, |
382 | struct page *page, dma_addr_t dma_addr, |
383 | enum dma_data_direction dir) |
384 | { |
385 | void *vaddr = page_address(page); |
386 | |
387 | /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ |
388 | if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && |
389 | dma_free_from_pool(dev, start: vaddr, size)) |
390 | return; |
391 | |
392 | if (dma_set_encrypted(dev, vaddr, size)) |
393 | return; |
394 | __dma_direct_free_pages(dev, page, size); |
395 | } |
396 | |
397 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ |
398 | defined(CONFIG_SWIOTLB) |
399 | void dma_direct_sync_sg_for_device(struct device *dev, |
400 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) |
401 | { |
402 | struct scatterlist *sg; |
403 | int i; |
404 | |
405 | for_each_sg(sgl, sg, nents, i) { |
406 | phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); |
407 | |
408 | swiotlb_sync_single_for_device(dev, addr: paddr, size: sg->length, dir); |
409 | |
410 | if (!dev_is_dma_coherent(dev)) |
411 | arch_sync_dma_for_device(paddr, size: sg->length, |
412 | dir); |
413 | } |
414 | } |
415 | #endif |
416 | |
417 | #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ |
418 | defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \ |
419 | defined(CONFIG_SWIOTLB) |
420 | void dma_direct_sync_sg_for_cpu(struct device *dev, |
421 | struct scatterlist *sgl, int nents, enum dma_data_direction dir) |
422 | { |
423 | struct scatterlist *sg; |
424 | int i; |
425 | |
426 | for_each_sg(sgl, sg, nents, i) { |
427 | phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); |
428 | |
429 | if (!dev_is_dma_coherent(dev)) |
430 | arch_sync_dma_for_cpu(paddr, size: sg->length, dir); |
431 | |
432 | swiotlb_sync_single_for_cpu(dev, addr: paddr, size: sg->length, dir); |
433 | |
434 | if (dir == DMA_FROM_DEVICE) |
435 | arch_dma_mark_clean(paddr, size: sg->length); |
436 | } |
437 | |
438 | if (!dev_is_dma_coherent(dev)) |
439 | arch_sync_dma_for_cpu_all(); |
440 | } |
441 | |
442 | /* |
443 | * Unmaps segments, except for ones marked as pci_p2pdma which do not |
444 | * require any further action as they contain a bus address. |
445 | */ |
446 | void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, |
447 | int nents, enum dma_data_direction dir, unsigned long attrs) |
448 | { |
449 | struct scatterlist *sg; |
450 | int i; |
451 | |
452 | for_each_sg(sgl, sg, nents, i) { |
453 | if (sg_dma_is_bus_address(sg)) |
454 | sg_dma_unmark_bus_address(sg); |
455 | else |
456 | dma_direct_unmap_page(dev, addr: sg->dma_address, |
457 | sg_dma_len(sg), dir, attrs); |
458 | } |
459 | } |
460 | #endif |
461 | |
462 | int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, |
463 | enum dma_data_direction dir, unsigned long attrs) |
464 | { |
465 | struct pci_p2pdma_map_state p2pdma_state = {}; |
466 | struct scatterlist *sg; |
467 | int i, ret; |
468 | |
469 | for_each_sg(sgl, sg, nents, i) { |
470 | switch (pci_p2pdma_state(state: &p2pdma_state, dev, page: sg_page(sg))) { |
471 | case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: |
472 | /* |
473 | * Any P2P mapping that traverses the PCI host bridge |
474 | * must be mapped with CPU physical address and not PCI |
475 | * bus addresses. |
476 | */ |
477 | break; |
478 | case PCI_P2PDMA_MAP_NONE: |
479 | sg->dma_address = dma_direct_map_page(dev, page: sg_page(sg), |
480 | offset: sg->offset, size: sg->length, dir, attrs); |
481 | if (sg->dma_address == DMA_MAPPING_ERROR) { |
482 | ret = -EIO; |
483 | goto out_unmap; |
484 | } |
485 | break; |
486 | case PCI_P2PDMA_MAP_BUS_ADDR: |
487 | sg->dma_address = pci_p2pdma_bus_addr_map(state: &p2pdma_state, |
488 | paddr: sg_phys(sg)); |
489 | sg_dma_mark_bus_address(sg); |
490 | continue; |
491 | default: |
492 | ret = -EREMOTEIO; |
493 | goto out_unmap; |
494 | } |
495 | sg_dma_len(sg) = sg->length; |
496 | } |
497 | |
498 | return nents; |
499 | |
500 | out_unmap: |
501 | dma_direct_unmap_sg(dev, sgl, nents: i, dir, attrs: attrs | DMA_ATTR_SKIP_CPU_SYNC); |
502 | return ret; |
503 | } |
504 | |
505 | dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, |
506 | size_t size, enum dma_data_direction dir, unsigned long attrs) |
507 | { |
508 | dma_addr_t dma_addr = paddr; |
509 | |
510 | if (unlikely(!dma_capable(dev, dma_addr, size, false))) { |
511 | dev_err_once(dev, |
512 | "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", |
513 | &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); |
514 | WARN_ON_ONCE(1); |
515 | return DMA_MAPPING_ERROR; |
516 | } |
517 | |
518 | return dma_addr; |
519 | } |
520 | |
521 | int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt, |
522 | void *cpu_addr, dma_addr_t dma_addr, size_t size, |
523 | unsigned long attrs) |
524 | { |
525 | struct page *page = dma_direct_to_page(dev, dma_addr); |
526 | int ret; |
527 | |
528 | ret = sg_alloc_table(sgt, 1, GFP_KERNEL); |
529 | if (!ret) |
530 | sg_set_page(sg: sgt->sgl, page, PAGE_ALIGN(size), offset: 0); |
531 | return ret; |
532 | } |
533 | |
534 | bool dma_direct_can_mmap(struct device *dev) |
535 | { |
536 | return dev_is_dma_coherent(dev) || |
537 | IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP); |
538 | } |
539 | |
540 | int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, |
541 | void *cpu_addr, dma_addr_t dma_addr, size_t size, |
542 | unsigned long attrs) |
543 | { |
544 | unsigned long user_count = vma_pages(vma); |
545 | unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; |
546 | unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); |
547 | int ret = -ENXIO; |
548 | |
549 | vma->vm_page_prot = dma_pgprot(dev, prot: vma->vm_page_prot, attrs); |
550 | if (force_dma_unencrypted(dev)) |
551 | vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); |
552 | |
553 | if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, ret: &ret)) |
554 | return ret; |
555 | if (dma_mmap_from_global_coherent(vma, cpu_addr, size, ret: &ret)) |
556 | return ret; |
557 | |
558 | if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) |
559 | return -ENXIO; |
560 | return remap_pfn_range(vma, addr: vma->vm_start, pfn: pfn + vma->vm_pgoff, |
561 | size: user_count << PAGE_SHIFT, vma->vm_page_prot); |
562 | } |
563 | |
564 | int dma_direct_supported(struct device *dev, u64 mask) |
565 | { |
566 | u64 min_mask = (max_pfn - 1) << PAGE_SHIFT; |
567 | |
568 | /* |
569 | * Because 32-bit DMA masks are so common we expect every architecture |
570 | * to be able to satisfy them - either by not supporting more physical |
571 | * memory, or by providing a ZONE_DMA32. If neither is the case, the |
572 | * architecture needs to use an IOMMU instead of the direct mapping. |
573 | */ |
574 | if (mask >= DMA_BIT_MASK(32)) |
575 | return 1; |
576 | |
577 | /* |
578 | * This check needs to be against the actual bit mask value, so use |
579 | * phys_to_dma_unencrypted() here so that the SME encryption mask isn't |
580 | * part of the check. |
581 | */ |
582 | if (IS_ENABLED(CONFIG_ZONE_DMA)) |
583 | min_mask = min_t(u64, min_mask, zone_dma_limit); |
584 | return mask >= phys_to_dma_unencrypted(dev, paddr: min_mask); |
585 | } |
586 | |
587 | static const struct bus_dma_region *dma_find_range(struct device *dev, |
588 | unsigned long start_pfn) |
589 | { |
590 | const struct bus_dma_region *m; |
591 | |
592 | for (m = dev->dma_range_map; PFN_DOWN(m->size); m++) { |
593 | unsigned long cpu_start_pfn = PFN_DOWN(m->cpu_start); |
594 | |
595 | if (start_pfn >= cpu_start_pfn && |
596 | start_pfn - cpu_start_pfn < PFN_DOWN(m->size)) |
597 | return m; |
598 | } |
599 | |
600 | return NULL; |
601 | } |
602 | |
603 | /* |
604 | * To check whether all ram resource ranges are covered by dma range map |
605 | * Returns 0 when further check is needed |
606 | * Returns 1 if there is some RAM range can't be covered by dma_range_map |
607 | */ |
608 | static int check_ram_in_range_map(unsigned long start_pfn, |
609 | unsigned long nr_pages, void *data) |
610 | { |
611 | unsigned long end_pfn = start_pfn + nr_pages; |
612 | struct device *dev = data; |
613 | |
614 | while (start_pfn < end_pfn) { |
615 | const struct bus_dma_region *bdr; |
616 | |
617 | bdr = dma_find_range(dev, start_pfn); |
618 | if (!bdr) |
619 | return 1; |
620 | |
621 | start_pfn = PFN_DOWN(bdr->cpu_start) + PFN_DOWN(bdr->size); |
622 | } |
623 | |
624 | return 0; |
625 | } |
626 | |
627 | bool dma_direct_all_ram_mapped(struct device *dev) |
628 | { |
629 | if (!dev->dma_range_map) |
630 | return true; |
631 | return !walk_system_ram_range(start_pfn: 0, PFN_DOWN(ULONG_MAX) + 1, arg: dev, |
632 | func: check_ram_in_range_map); |
633 | } |
634 | |
635 | size_t dma_direct_max_mapping_size(struct device *dev) |
636 | { |
637 | /* If SWIOTLB is active, use its maximum mapping size */ |
638 | if (is_swiotlb_active(dev) && |
639 | (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev))) |
640 | return swiotlb_max_mapping_size(dev); |
641 | return SIZE_MAX; |
642 | } |
643 | |
644 | bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr) |
645 | { |
646 | return !dev_is_dma_coherent(dev) || |
647 | swiotlb_find_pool(dev, paddr: dma_to_phys(dev, dma_addr)); |
648 | } |
649 | |
650 | /** |
651 | * dma_direct_set_offset - Assign scalar offset for a single DMA range. |
652 | * @dev: device pointer; needed to "own" the alloced memory. |
653 | * @cpu_start: beginning of memory region covered by this offset. |
654 | * @dma_start: beginning of DMA/PCI region covered by this offset. |
655 | * @size: size of the region. |
656 | * |
657 | * This is for the simple case of a uniform offset which cannot |
658 | * be discovered by "dma-ranges". |
659 | * |
660 | * It returns -ENOMEM if out of memory, -EINVAL if a map |
661 | * already exists, 0 otherwise. |
662 | * |
663 | * Note: any call to this from a driver is a bug. The mapping needs |
664 | * to be described by the device tree or other firmware interfaces. |
665 | */ |
666 | int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, |
667 | dma_addr_t dma_start, u64 size) |
668 | { |
669 | struct bus_dma_region *map; |
670 | u64 offset = (u64)cpu_start - (u64)dma_start; |
671 | |
672 | if (dev->dma_range_map) { |
673 | dev_err(dev, "attempt to add DMA range to existing map\n"); |
674 | return -EINVAL; |
675 | } |
676 | |
677 | if (!offset) |
678 | return 0; |
679 | |
680 | map = kcalloc(2, sizeof(*map), GFP_KERNEL); |
681 | if (!map) |
682 | return -ENOMEM; |
683 | map[0].cpu_start = cpu_start; |
684 | map[0].dma_start = dma_start; |
685 | map[0].size = size; |
686 | dev->dma_range_map = map; |
687 | return 0; |
688 | } |
689 |
Definitions
- zone_dma_limit
- phys_to_dma_direct
- dma_direct_to_page
- dma_direct_get_required_mask
- dma_direct_optimal_gfp_mask
- dma_coherent_ok
- dma_set_decrypted
- dma_set_encrypted
- __dma_direct_free_pages
- dma_direct_alloc_swiotlb
- __dma_direct_alloc_pages
- dma_direct_use_pool
- dma_direct_alloc_from_pool
- dma_direct_alloc_no_mapping
- dma_direct_alloc
- dma_direct_free
- dma_direct_alloc_pages
- dma_direct_free_pages
- dma_direct_sync_sg_for_device
- dma_direct_sync_sg_for_cpu
- dma_direct_unmap_sg
- dma_direct_map_sg
- dma_direct_map_resource
- dma_direct_get_sgtable
- dma_direct_can_mmap
- dma_direct_mmap
- dma_direct_supported
- dma_find_range
- check_ram_in_range_map
- dma_direct_all_ram_mapped
- dma_direct_max_mapping_size
- dma_direct_need_sync
Improve your Profiling and Debugging skills
Find out more