1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Helpers for IOMMU drivers implementing SVA |
4 | */ |
5 | #include <linux/mmu_context.h> |
6 | #include <linux/mutex.h> |
7 | #include <linux/sched/mm.h> |
8 | #include <linux/iommu.h> |
9 | |
10 | #include "iommu-priv.h" |
11 | |
12 | static DEFINE_MUTEX(iommu_sva_lock); |
13 | |
14 | /* Allocate a PASID for the mm within range (inclusive) */ |
15 | static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct device *dev) |
16 | { |
17 | struct iommu_mm_data *iommu_mm; |
18 | ioasid_t pasid; |
19 | |
20 | lockdep_assert_held(&iommu_sva_lock); |
21 | |
22 | if (!arch_pgtable_dma_compat(mm)) |
23 | return ERR_PTR(error: -EBUSY); |
24 | |
25 | iommu_mm = mm->iommu_mm; |
26 | /* Is a PASID already associated with this mm? */ |
27 | if (iommu_mm) { |
28 | if (iommu_mm->pasid >= dev->iommu->max_pasids) |
29 | return ERR_PTR(error: -EOVERFLOW); |
30 | return iommu_mm; |
31 | } |
32 | |
33 | iommu_mm = kzalloc(size: sizeof(struct iommu_mm_data), GFP_KERNEL); |
34 | if (!iommu_mm) |
35 | return ERR_PTR(error: -ENOMEM); |
36 | |
37 | pasid = iommu_alloc_global_pasid(dev); |
38 | if (pasid == IOMMU_PASID_INVALID) { |
39 | kfree(objp: iommu_mm); |
40 | return ERR_PTR(error: -ENOSPC); |
41 | } |
42 | iommu_mm->pasid = pasid; |
43 | INIT_LIST_HEAD(list: &iommu_mm->sva_domains); |
44 | INIT_LIST_HEAD(list: &iommu_mm->sva_handles); |
45 | /* |
46 | * Make sure the write to mm->iommu_mm is not reordered in front of |
47 | * initialization to iommu_mm fields. If it does, readers may see a |
48 | * valid iommu_mm with uninitialized values. |
49 | */ |
50 | smp_store_release(&mm->iommu_mm, iommu_mm); |
51 | return iommu_mm; |
52 | } |
53 | |
54 | /** |
55 | * iommu_sva_bind_device() - Bind a process address space to a device |
56 | * @dev: the device |
57 | * @mm: the mm to bind, caller must hold a reference to mm_users |
58 | * |
59 | * Create a bond between device and address space, allowing the device to |
60 | * access the mm using the PASID returned by iommu_sva_get_pasid(). If a |
61 | * bond already exists between @device and @mm, an additional internal |
62 | * reference is taken. Caller must call iommu_sva_unbind_device() |
63 | * to release each reference. |
64 | * |
65 | * iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA) must be called first, to |
66 | * initialize the required SVA features. |
67 | * |
68 | * On error, returns an ERR_PTR value. |
69 | */ |
70 | struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) |
71 | { |
72 | struct iommu_mm_data *iommu_mm; |
73 | struct iommu_domain *domain; |
74 | struct iommu_sva *handle; |
75 | int ret; |
76 | |
77 | mutex_lock(&iommu_sva_lock); |
78 | |
79 | /* Allocate mm->pasid if necessary. */ |
80 | iommu_mm = iommu_alloc_mm_data(mm, dev); |
81 | if (IS_ERR(ptr: iommu_mm)) { |
82 | ret = PTR_ERR(ptr: iommu_mm); |
83 | goto out_unlock; |
84 | } |
85 | |
86 | list_for_each_entry(handle, &mm->iommu_mm->sva_handles, handle_item) { |
87 | if (handle->dev == dev) { |
88 | refcount_inc(r: &handle->users); |
89 | mutex_unlock(lock: &iommu_sva_lock); |
90 | return handle; |
91 | } |
92 | } |
93 | |
94 | handle = kzalloc(size: sizeof(*handle), GFP_KERNEL); |
95 | if (!handle) { |
96 | ret = -ENOMEM; |
97 | goto out_unlock; |
98 | } |
99 | |
100 | /* Search for an existing domain. */ |
101 | list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) { |
102 | ret = iommu_attach_device_pasid(domain, dev, pasid: iommu_mm->pasid); |
103 | if (!ret) { |
104 | domain->users++; |
105 | goto out; |
106 | } |
107 | } |
108 | |
109 | /* Allocate a new domain and set it on device pasid. */ |
110 | domain = iommu_sva_domain_alloc(dev, mm); |
111 | if (!domain) { |
112 | ret = -ENOMEM; |
113 | goto out_free_handle; |
114 | } |
115 | |
116 | ret = iommu_attach_device_pasid(domain, dev, pasid: iommu_mm->pasid); |
117 | if (ret) |
118 | goto out_free_domain; |
119 | domain->users = 1; |
120 | list_add(new: &domain->next, head: &mm->iommu_mm->sva_domains); |
121 | |
122 | out: |
123 | refcount_set(r: &handle->users, n: 1); |
124 | list_add(new: &handle->handle_item, head: &mm->iommu_mm->sva_handles); |
125 | mutex_unlock(lock: &iommu_sva_lock); |
126 | handle->dev = dev; |
127 | handle->domain = domain; |
128 | return handle; |
129 | |
130 | out_free_domain: |
131 | iommu_domain_free(domain); |
132 | out_free_handle: |
133 | kfree(objp: handle); |
134 | out_unlock: |
135 | mutex_unlock(lock: &iommu_sva_lock); |
136 | return ERR_PTR(error: ret); |
137 | } |
138 | EXPORT_SYMBOL_GPL(iommu_sva_bind_device); |
139 | |
140 | /** |
141 | * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device |
142 | * @handle: the handle returned by iommu_sva_bind_device() |
143 | * |
144 | * Put reference to a bond between device and address space. The device should |
145 | * not be issuing any more transaction for this PASID. All outstanding page |
146 | * requests for this PASID must have been flushed to the IOMMU. |
147 | */ |
148 | void iommu_sva_unbind_device(struct iommu_sva *handle) |
149 | { |
150 | struct iommu_domain *domain = handle->domain; |
151 | struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm; |
152 | struct device *dev = handle->dev; |
153 | |
154 | mutex_lock(&iommu_sva_lock); |
155 | if (!refcount_dec_and_test(r: &handle->users)) { |
156 | mutex_unlock(lock: &iommu_sva_lock); |
157 | return; |
158 | } |
159 | list_del(entry: &handle->handle_item); |
160 | |
161 | iommu_detach_device_pasid(domain, dev, pasid: iommu_mm->pasid); |
162 | if (--domain->users == 0) { |
163 | list_del(entry: &domain->next); |
164 | iommu_domain_free(domain); |
165 | } |
166 | mutex_unlock(lock: &iommu_sva_lock); |
167 | kfree(objp: handle); |
168 | } |
169 | EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); |
170 | |
171 | u32 iommu_sva_get_pasid(struct iommu_sva *handle) |
172 | { |
173 | struct iommu_domain *domain = handle->domain; |
174 | |
175 | return mm_get_enqcmd_pasid(mm: domain->mm); |
176 | } |
177 | EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); |
178 | |
179 | void mm_pasid_drop(struct mm_struct *mm) |
180 | { |
181 | struct iommu_mm_data *iommu_mm = mm->iommu_mm; |
182 | |
183 | if (!iommu_mm) |
184 | return; |
185 | |
186 | iommu_free_global_pasid(pasid: iommu_mm->pasid); |
187 | kfree(objp: iommu_mm); |
188 | } |
189 | |
190 | /* |
191 | * I/O page fault handler for SVA |
192 | */ |
193 | static enum iommu_page_response_code |
194 | iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) |
195 | { |
196 | vm_fault_t ret; |
197 | struct vm_area_struct *vma; |
198 | unsigned int access_flags = 0; |
199 | unsigned int fault_flags = FAULT_FLAG_REMOTE; |
200 | struct iommu_fault_page_request *prm = &fault->prm; |
201 | enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID; |
202 | |
203 | if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID)) |
204 | return status; |
205 | |
206 | if (!mmget_not_zero(mm)) |
207 | return status; |
208 | |
209 | mmap_read_lock(mm); |
210 | |
211 | vma = vma_lookup(mm, addr: prm->addr); |
212 | if (!vma) |
213 | /* Unmapped area */ |
214 | goto out_put_mm; |
215 | |
216 | if (prm->perm & IOMMU_FAULT_PERM_READ) |
217 | access_flags |= VM_READ; |
218 | |
219 | if (prm->perm & IOMMU_FAULT_PERM_WRITE) { |
220 | access_flags |= VM_WRITE; |
221 | fault_flags |= FAULT_FLAG_WRITE; |
222 | } |
223 | |
224 | if (prm->perm & IOMMU_FAULT_PERM_EXEC) { |
225 | access_flags |= VM_EXEC; |
226 | fault_flags |= FAULT_FLAG_INSTRUCTION; |
227 | } |
228 | |
229 | if (!(prm->perm & IOMMU_FAULT_PERM_PRIV)) |
230 | fault_flags |= FAULT_FLAG_USER; |
231 | |
232 | if (access_flags & ~vma->vm_flags) |
233 | /* Access fault */ |
234 | goto out_put_mm; |
235 | |
236 | ret = handle_mm_fault(vma, address: prm->addr, flags: fault_flags, NULL); |
237 | status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID : |
238 | IOMMU_PAGE_RESP_SUCCESS; |
239 | |
240 | out_put_mm: |
241 | mmap_read_unlock(mm); |
242 | mmput(mm); |
243 | |
244 | return status; |
245 | } |
246 | |
247 | static void iommu_sva_handle_iopf(struct work_struct *work) |
248 | { |
249 | struct iopf_fault *iopf; |
250 | struct iopf_group *group; |
251 | enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; |
252 | |
253 | group = container_of(work, struct iopf_group, work); |
254 | list_for_each_entry(iopf, &group->faults, list) { |
255 | /* |
256 | * For the moment, errors are sticky: don't handle subsequent |
257 | * faults in the group if there is an error. |
258 | */ |
259 | if (status != IOMMU_PAGE_RESP_SUCCESS) |
260 | break; |
261 | |
262 | status = iommu_sva_handle_mm(fault: &iopf->fault, mm: group->domain->mm); |
263 | } |
264 | |
265 | iopf_group_response(group, status); |
266 | iopf_free_group(group); |
267 | } |
268 | |
269 | static int iommu_sva_iopf_handler(struct iopf_group *group) |
270 | { |
271 | struct iommu_fault_param *fault_param = group->fault_param; |
272 | |
273 | INIT_WORK(&group->work, iommu_sva_handle_iopf); |
274 | if (!queue_work(wq: fault_param->queue->wq, work: &group->work)) |
275 | return -EBUSY; |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, |
281 | struct mm_struct *mm) |
282 | { |
283 | const struct iommu_ops *ops = dev_iommu_ops(dev); |
284 | struct iommu_domain *domain; |
285 | |
286 | domain = ops->domain_alloc(IOMMU_DOMAIN_SVA); |
287 | if (!domain) |
288 | return NULL; |
289 | |
290 | domain->type = IOMMU_DOMAIN_SVA; |
291 | mmgrab(mm); |
292 | domain->mm = mm; |
293 | domain->owner = ops; |
294 | domain->iopf_handler = iommu_sva_iopf_handler; |
295 | |
296 | return domain; |
297 | } |
298 | |