1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/cpu.h> |
3 | #include <linux/dma-direct.h> |
4 | #include <linux/dma-map-ops.h> |
5 | #include <linux/gfp.h> |
6 | #include <linux/highmem.h> |
7 | #include <linux/export.h> |
8 | #include <linux/memblock.h> |
9 | #include <linux/of_address.h> |
10 | #include <linux/slab.h> |
11 | #include <linux/types.h> |
12 | #include <linux/vmalloc.h> |
13 | #include <linux/swiotlb.h> |
14 | |
15 | #include <xen/xen.h> |
16 | #include <xen/interface/grant_table.h> |
17 | #include <xen/interface/memory.h> |
18 | #include <xen/page.h> |
19 | #include <xen/xen-ops.h> |
20 | #include <xen/swiotlb-xen.h> |
21 | |
22 | #include <asm/cacheflush.h> |
23 | #include <asm/xen/hypercall.h> |
24 | #include <asm/xen/interface.h> |
25 | |
26 | static gfp_t xen_swiotlb_gfp(void) |
27 | { |
28 | phys_addr_t base; |
29 | u64 i; |
30 | |
31 | for_each_mem_range(i, &base, NULL) { |
32 | if (base < (phys_addr_t)0xffffffff) { |
33 | if (IS_ENABLED(CONFIG_ZONE_DMA32)) |
34 | return __GFP_DMA32; |
35 | return __GFP_DMA; |
36 | } |
37 | } |
38 | |
39 | return GFP_KERNEL; |
40 | } |
41 | |
42 | static bool hypercall_cflush = false; |
43 | |
44 | /* buffers in highmem or foreign pages cannot cross page boundaries */ |
45 | static void dma_cache_maint(struct device *dev, dma_addr_t handle, |
46 | size_t size, u32 op) |
47 | { |
48 | struct gnttab_cache_flush cflush; |
49 | |
50 | cflush.offset = xen_offset_in_page(handle); |
51 | cflush.op = op; |
52 | handle &= XEN_PAGE_MASK; |
53 | |
54 | do { |
55 | cflush.a.dev_bus_addr = dma_to_phys(dev, dma_addr: handle); |
56 | |
57 | if (size + cflush.offset > XEN_PAGE_SIZE) |
58 | cflush.length = XEN_PAGE_SIZE - cflush.offset; |
59 | else |
60 | cflush.length = size; |
61 | |
62 | HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, uop: &cflush, count: 1); |
63 | |
64 | cflush.offset = 0; |
65 | handle += cflush.length; |
66 | size -= cflush.length; |
67 | } while (size); |
68 | } |
69 | |
70 | /* |
71 | * Dom0 is mapped 1:1, and while the Linux page can span across multiple Xen |
72 | * pages, it is not possible for it to contain a mix of local and foreign Xen |
73 | * pages. Calling pfn_valid on a foreign mfn will always return false, so if |
74 | * pfn_valid returns true the pages is local and we can use the native |
75 | * dma-direct functions, otherwise we call the Xen specific version. |
76 | */ |
77 | void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, |
78 | size_t size, enum dma_data_direction dir) |
79 | { |
80 | if (dir != DMA_TO_DEVICE) |
81 | dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); |
82 | } |
83 | |
84 | void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, |
85 | size_t size, enum dma_data_direction dir) |
86 | { |
87 | if (dir == DMA_FROM_DEVICE) |
88 | dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); |
89 | else |
90 | dma_cache_maint(dev, handle, size, GNTTAB_CACHE_CLEAN); |
91 | } |
92 | |
93 | bool xen_arch_need_swiotlb(struct device *dev, |
94 | phys_addr_t phys, |
95 | dma_addr_t dev_addr) |
96 | { |
97 | unsigned int xen_pfn = XEN_PFN_DOWN(phys); |
98 | unsigned int bfn = XEN_PFN_DOWN(dma_to_phys(dev, dev_addr)); |
99 | |
100 | /* |
101 | * The swiotlb buffer should be used if |
102 | * - Xen doesn't have the cache flush hypercall |
103 | * - The Linux page refers to foreign memory |
104 | * - The device doesn't support coherent DMA request |
105 | * |
106 | * The Linux page may be spanned acrros multiple Xen page, although |
107 | * it's not possible to have a mix of local and foreign Xen page. |
108 | * Furthermore, range_straddles_page_boundary is already checking |
109 | * if buffer is physically contiguous in the host RAM. |
110 | * |
111 | * Therefore we only need to check the first Xen page to know if we |
112 | * require a bounce buffer because the device doesn't support coherent |
113 | * memory and we are not able to flush the cache. |
114 | */ |
115 | return (!hypercall_cflush && (xen_pfn != bfn) && |
116 | !dev_is_dma_coherent(dev)); |
117 | } |
118 | |
119 | static int __init xen_mm_init(void) |
120 | { |
121 | struct gnttab_cache_flush cflush; |
122 | int rc; |
123 | |
124 | if (!xen_swiotlb_detect()) |
125 | return 0; |
126 | |
127 | /* we can work with the default swiotlb */ |
128 | rc = swiotlb_init_late(size: swiotlb_size_or_default(), |
129 | gfp_mask: xen_swiotlb_gfp(), NULL); |
130 | if (rc < 0) |
131 | return rc; |
132 | |
133 | cflush.op = 0; |
134 | cflush.a.dev_bus_addr = 0; |
135 | cflush.offset = 0; |
136 | cflush.length = 0; |
137 | if (HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, uop: &cflush, count: 1) != -ENOSYS) |
138 | hypercall_cflush = true; |
139 | return 0; |
140 | } |
141 | arch_initcall(xen_mm_init); |
142 | |