1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | #include <linux/memblock.h> |
3 | #include <linux/gfp.h> |
4 | #include <linux/export.h> |
5 | #include <linux/spinlock.h> |
6 | #include <linux/slab.h> |
7 | #include <linux/types.h> |
8 | #include <linux/dma-mapping.h> |
9 | #include <linux/vmalloc.h> |
10 | #include <linux/swiotlb.h> |
11 | |
12 | #include <xen/xen.h> |
13 | #include <xen/interface/memory.h> |
14 | #include <xen/grant_table.h> |
15 | #include <xen/page.h> |
16 | #include <xen/swiotlb-xen.h> |
17 | |
18 | #include <asm/cacheflush.h> |
19 | #include <asm/xen/hypercall.h> |
20 | #include <asm/xen/interface.h> |
21 | |
22 | struct xen_p2m_entry { |
23 | unsigned long pfn; |
24 | unsigned long mfn; |
25 | unsigned long nr_pages; |
26 | struct rb_node rbnode_phys; |
27 | }; |
28 | |
29 | static rwlock_t p2m_lock; |
30 | struct rb_root phys_to_mach = RB_ROOT; |
31 | EXPORT_SYMBOL_GPL(phys_to_mach); |
32 | |
33 | static int xen_add_phys_to_mach_entry(struct xen_p2m_entry *new) |
34 | { |
35 | struct rb_node **link = &phys_to_mach.rb_node; |
36 | struct rb_node *parent = NULL; |
37 | struct xen_p2m_entry *entry; |
38 | int rc = 0; |
39 | |
40 | while (*link) { |
41 | parent = *link; |
42 | entry = rb_entry(parent, struct xen_p2m_entry, rbnode_phys); |
43 | |
44 | if (new->pfn == entry->pfn) |
45 | goto err_out; |
46 | |
47 | if (new->pfn < entry->pfn) |
48 | link = &(*link)->rb_left; |
49 | else |
50 | link = &(*link)->rb_right; |
51 | } |
52 | rb_link_node(node: &new->rbnode_phys, parent, rb_link: link); |
53 | rb_insert_color(&new->rbnode_phys, &phys_to_mach); |
54 | goto out; |
55 | |
56 | err_out: |
57 | rc = -EINVAL; |
58 | pr_warn("%s: cannot add pfn=%pa -> mfn=%pa: pfn=%pa -> mfn=%pa already exists\n" , |
59 | __func__, &new->pfn, &new->mfn, &entry->pfn, &entry->mfn); |
60 | out: |
61 | return rc; |
62 | } |
63 | |
64 | unsigned long __pfn_to_mfn(unsigned long pfn) |
65 | { |
66 | struct rb_node *n; |
67 | struct xen_p2m_entry *entry; |
68 | unsigned long irqflags; |
69 | |
70 | read_lock_irqsave(&p2m_lock, irqflags); |
71 | n = phys_to_mach.rb_node; |
72 | while (n) { |
73 | entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); |
74 | if (entry->pfn <= pfn && |
75 | entry->pfn + entry->nr_pages > pfn) { |
76 | unsigned long mfn = entry->mfn + (pfn - entry->pfn); |
77 | read_unlock_irqrestore(&p2m_lock, irqflags); |
78 | return mfn; |
79 | } |
80 | if (pfn < entry->pfn) |
81 | n = n->rb_left; |
82 | else |
83 | n = n->rb_right; |
84 | } |
85 | read_unlock_irqrestore(&p2m_lock, irqflags); |
86 | |
87 | return INVALID_P2M_ENTRY; |
88 | } |
89 | EXPORT_SYMBOL_GPL(__pfn_to_mfn); |
90 | |
91 | int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, |
92 | struct gnttab_map_grant_ref *kmap_ops, |
93 | struct page **pages, unsigned int count) |
94 | { |
95 | int i; |
96 | |
97 | for (i = 0; i < count; i++) { |
98 | struct gnttab_unmap_grant_ref unmap; |
99 | int rc; |
100 | |
101 | if (map_ops[i].status) |
102 | continue; |
103 | if (likely(set_phys_to_machine(map_ops[i].host_addr >> XEN_PAGE_SHIFT, |
104 | map_ops[i].dev_bus_addr >> XEN_PAGE_SHIFT))) |
105 | continue; |
106 | |
107 | /* |
108 | * Signal an error for this slot. This in turn requires |
109 | * immediate unmapping. |
110 | */ |
111 | map_ops[i].status = GNTST_general_error; |
112 | unmap.host_addr = map_ops[i].host_addr, |
113 | unmap.handle = map_ops[i].handle; |
114 | map_ops[i].handle = INVALID_GRANT_HANDLE; |
115 | if (map_ops[i].flags & GNTMAP_device_map) |
116 | unmap.dev_bus_addr = map_ops[i].dev_bus_addr; |
117 | else |
118 | unmap.dev_bus_addr = 0; |
119 | |
120 | /* |
121 | * Pre-populate the status field, to be recognizable in |
122 | * the log message below. |
123 | */ |
124 | unmap.status = 1; |
125 | |
126 | rc = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, |
127 | uop: &unmap, count: 1); |
128 | if (rc || unmap.status != GNTST_okay) |
129 | pr_err_once("gnttab unmap failed: rc=%d st=%d\n" , |
130 | rc, unmap.status); |
131 | } |
132 | |
133 | return 0; |
134 | } |
135 | |
136 | int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, |
137 | struct gnttab_unmap_grant_ref *kunmap_ops, |
138 | struct page **pages, unsigned int count) |
139 | { |
140 | int i; |
141 | |
142 | for (i = 0; i < count; i++) { |
143 | set_phys_to_machine(pfn: unmap_ops[i].host_addr >> XEN_PAGE_SHIFT, |
144 | INVALID_P2M_ENTRY); |
145 | } |
146 | |
147 | return 0; |
148 | } |
149 | |
150 | bool __set_phys_to_machine_multi(unsigned long pfn, |
151 | unsigned long mfn, unsigned long nr_pages) |
152 | { |
153 | int rc; |
154 | unsigned long irqflags; |
155 | struct xen_p2m_entry *p2m_entry; |
156 | struct rb_node *n; |
157 | |
158 | if (mfn == INVALID_P2M_ENTRY) { |
159 | write_lock_irqsave(&p2m_lock, irqflags); |
160 | n = phys_to_mach.rb_node; |
161 | while (n) { |
162 | p2m_entry = rb_entry(n, struct xen_p2m_entry, rbnode_phys); |
163 | if (p2m_entry->pfn <= pfn && |
164 | p2m_entry->pfn + p2m_entry->nr_pages > pfn) { |
165 | rb_erase(&p2m_entry->rbnode_phys, &phys_to_mach); |
166 | write_unlock_irqrestore(&p2m_lock, irqflags); |
167 | kfree(objp: p2m_entry); |
168 | return true; |
169 | } |
170 | if (pfn < p2m_entry->pfn) |
171 | n = n->rb_left; |
172 | else |
173 | n = n->rb_right; |
174 | } |
175 | write_unlock_irqrestore(&p2m_lock, irqflags); |
176 | return true; |
177 | } |
178 | |
179 | p2m_entry = kzalloc(size: sizeof(*p2m_entry), GFP_NOWAIT); |
180 | if (!p2m_entry) |
181 | return false; |
182 | |
183 | p2m_entry->pfn = pfn; |
184 | p2m_entry->nr_pages = nr_pages; |
185 | p2m_entry->mfn = mfn; |
186 | |
187 | write_lock_irqsave(&p2m_lock, irqflags); |
188 | rc = xen_add_phys_to_mach_entry(new: p2m_entry); |
189 | if (rc < 0) { |
190 | write_unlock_irqrestore(&p2m_lock, irqflags); |
191 | kfree(objp: p2m_entry); |
192 | return false; |
193 | } |
194 | write_unlock_irqrestore(&p2m_lock, irqflags); |
195 | return true; |
196 | } |
197 | EXPORT_SYMBOL_GPL(__set_phys_to_machine_multi); |
198 | |
199 | bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn) |
200 | { |
201 | return __set_phys_to_machine_multi(pfn, mfn, 1); |
202 | } |
203 | EXPORT_SYMBOL_GPL(__set_phys_to_machine); |
204 | |
205 | static int p2m_init(void) |
206 | { |
207 | rwlock_init(&p2m_lock); |
208 | return 0; |
209 | } |
210 | arch_initcall(p2m_init); |
211 | |