1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Implementation of the IOMMU SVA API for the ARM SMMUv3
4 */
5
6#include <linux/mm.h>
7#include <linux/mmu_context.h>
8#include <linux/mmu_notifier.h>
9#include <linux/sched/mm.h>
10#include <linux/slab.h>
11
12#include "arm-smmu-v3.h"
13#include "../../io-pgtable-arm.h"
14
15struct arm_smmu_mmu_notifier {
16 struct mmu_notifier mn;
17 struct arm_smmu_ctx_desc *cd;
18 bool cleared;
19 refcount_t refs;
20 struct list_head list;
21 struct arm_smmu_domain *domain;
22};
23
24#define mn_to_smmu(mn) container_of(mn, struct arm_smmu_mmu_notifier, mn)
25
26struct arm_smmu_bond {
27 struct mm_struct *mm;
28 struct arm_smmu_mmu_notifier *smmu_mn;
29 struct list_head list;
30};
31
32#define sva_to_bond(handle) \
33 container_of(handle, struct arm_smmu_bond, sva)
34
35static DEFINE_MUTEX(sva_lock);
36
37/*
38 * Write the CD to the CD tables for all masters that this domain is attached
39 * to. Note that this is only used to update existing CD entries in the target
40 * CD table, for which it's assumed that arm_smmu_write_ctx_desc can't fail.
41 */
42static void arm_smmu_update_ctx_desc_devices(struct arm_smmu_domain *smmu_domain,
43 int ssid,
44 struct arm_smmu_ctx_desc *cd)
45{
46 struct arm_smmu_master *master;
47 unsigned long flags;
48
49 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
50 list_for_each_entry(master, &smmu_domain->devices, domain_head) {
51 arm_smmu_write_ctx_desc(smmu_master: master, ssid, cd);
52 }
53 spin_unlock_irqrestore(lock: &smmu_domain->devices_lock, flags);
54}
55
56/*
57 * Check if the CPU ASID is available on the SMMU side. If a private context
58 * descriptor is using it, try to replace it.
59 */
60static struct arm_smmu_ctx_desc *
61arm_smmu_share_asid(struct mm_struct *mm, u16 asid)
62{
63 int ret;
64 u32 new_asid;
65 struct arm_smmu_ctx_desc *cd;
66 struct arm_smmu_device *smmu;
67 struct arm_smmu_domain *smmu_domain;
68
69 cd = xa_load(&arm_smmu_asid_xa, index: asid);
70 if (!cd)
71 return NULL;
72
73 if (cd->mm) {
74 if (WARN_ON(cd->mm != mm))
75 return ERR_PTR(error: -EINVAL);
76 /* All devices bound to this mm use the same cd struct. */
77 refcount_inc(r: &cd->refs);
78 return cd;
79 }
80
81 smmu_domain = container_of(cd, struct arm_smmu_domain, cd);
82 smmu = smmu_domain->smmu;
83
84 ret = xa_alloc(xa: &arm_smmu_asid_xa, id: &new_asid, entry: cd,
85 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
86 if (ret)
87 return ERR_PTR(error: -ENOSPC);
88 /*
89 * Race with unmap: TLB invalidations will start targeting the new ASID,
90 * which isn't assigned yet. We'll do an invalidate-all on the old ASID
91 * later, so it doesn't matter.
92 */
93 cd->asid = new_asid;
94 /*
95 * Update ASID and invalidate CD in all associated masters. There will
96 * be some overlap between use of both ASIDs, until we invalidate the
97 * TLB.
98 */
99 arm_smmu_update_ctx_desc_devices(smmu_domain, IOMMU_NO_PASID, cd);
100
101 /* Invalidate TLB entries previously associated with that context */
102 arm_smmu_tlb_inv_asid(smmu, asid);
103
104 xa_erase(&arm_smmu_asid_xa, index: asid);
105 return NULL;
106}
107
108static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm)
109{
110 u16 asid;
111 int err = 0;
112 u64 tcr, par, reg;
113 struct arm_smmu_ctx_desc *cd;
114 struct arm_smmu_ctx_desc *ret = NULL;
115
116 /* Don't free the mm until we release the ASID */
117 mmgrab(mm);
118
119 asid = arm64_mm_context_get(mm);
120 if (!asid) {
121 err = -ESRCH;
122 goto out_drop_mm;
123 }
124
125 cd = kzalloc(size: sizeof(*cd), GFP_KERNEL);
126 if (!cd) {
127 err = -ENOMEM;
128 goto out_put_context;
129 }
130
131 refcount_set(r: &cd->refs, n: 1);
132
133 mutex_lock(&arm_smmu_asid_lock);
134 ret = arm_smmu_share_asid(mm, asid);
135 if (ret) {
136 mutex_unlock(lock: &arm_smmu_asid_lock);
137 goto out_free_cd;
138 }
139
140 err = xa_insert(xa: &arm_smmu_asid_xa, index: asid, entry: cd, GFP_KERNEL);
141 mutex_unlock(lock: &arm_smmu_asid_lock);
142
143 if (err)
144 goto out_free_asid;
145
146 tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) |
147 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) |
148 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) |
149 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) |
150 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64;
151
152 switch (PAGE_SIZE) {
153 case SZ_4K:
154 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K);
155 break;
156 case SZ_16K:
157 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K);
158 break;
159 case SZ_64K:
160 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K);
161 break;
162 default:
163 WARN_ON(1);
164 err = -EINVAL;
165 goto out_free_asid;
166 }
167
168 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
169 par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
170 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par);
171
172 cd->ttbr = virt_to_phys(address: mm->pgd);
173 cd->tcr = tcr;
174 /*
175 * MAIR value is pretty much constant and global, so we can just get it
176 * from the current CPU register
177 */
178 cd->mair = read_sysreg(mair_el1);
179 cd->asid = asid;
180 cd->mm = mm;
181
182 return cd;
183
184out_free_asid:
185 arm_smmu_free_asid(cd);
186out_free_cd:
187 kfree(objp: cd);
188out_put_context:
189 arm64_mm_context_put(mm);
190out_drop_mm:
191 mmdrop(mm);
192 return err < 0 ? ERR_PTR(error: err) : ret;
193}
194
195static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd)
196{
197 if (arm_smmu_free_asid(cd)) {
198 /* Unpin ASID */
199 arm64_mm_context_put(cd->mm);
200 mmdrop(mm: cd->mm);
201 kfree(objp: cd);
202 }
203}
204
205/*
206 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this
207 * is used as a threshold to replace per-page TLBI commands to issue in the
208 * command queue with an address-space TLBI command, when SMMU w/o a range
209 * invalidation feature handles too many per-page TLBI commands, which will
210 * otherwise result in a soft lockup.
211 */
212#define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3))
213
214static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
215 struct mm_struct *mm,
216 unsigned long start,
217 unsigned long end)
218{
219 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
220 struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
221 size_t size;
222
223 /*
224 * The mm_types defines vm_end as the first byte after the end address,
225 * different from IOMMU subsystem using the last address of an address
226 * range. So do a simple translation here by calculating size correctly.
227 */
228 size = end - start;
229 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) {
230 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE)
231 size = 0;
232 } else {
233 if (size == ULONG_MAX)
234 size = 0;
235 }
236
237 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) {
238 if (!size)
239 arm_smmu_tlb_inv_asid(smmu: smmu_domain->smmu,
240 asid: smmu_mn->cd->asid);
241 else
242 arm_smmu_tlb_inv_range_asid(iova: start, size,
243 asid: smmu_mn->cd->asid,
244 PAGE_SIZE, leaf: false,
245 smmu_domain);
246 }
247
248 arm_smmu_atc_inv_domain(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), iova: start,
249 size);
250}
251
252static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
253{
254 struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn);
255 struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
256
257 mutex_lock(&sva_lock);
258 if (smmu_mn->cleared) {
259 mutex_unlock(lock: &sva_lock);
260 return;
261 }
262
263 /*
264 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
265 * but disable translation.
266 */
267 arm_smmu_update_ctx_desc_devices(smmu_domain, ssid: mm_get_enqcmd_pasid(mm),
268 cd: &quiet_cd);
269
270 arm_smmu_tlb_inv_asid(smmu: smmu_domain->smmu, asid: smmu_mn->cd->asid);
271 arm_smmu_atc_inv_domain(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), iova: 0, size: 0);
272
273 smmu_mn->cleared = true;
274 mutex_unlock(lock: &sva_lock);
275}
276
277static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
278{
279 kfree(mn_to_smmu(mn));
280}
281
282static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
283 .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs,
284 .release = arm_smmu_mm_release,
285 .free_notifier = arm_smmu_mmu_notifier_free,
286};
287
288/* Allocate or get existing MMU notifier for this {domain, mm} pair */
289static struct arm_smmu_mmu_notifier *
290arm_smmu_mmu_notifier_get(struct arm_smmu_domain *smmu_domain,
291 struct mm_struct *mm)
292{
293 int ret;
294 struct arm_smmu_ctx_desc *cd;
295 struct arm_smmu_mmu_notifier *smmu_mn;
296
297 list_for_each_entry(smmu_mn, &smmu_domain->mmu_notifiers, list) {
298 if (smmu_mn->mn.mm == mm) {
299 refcount_inc(r: &smmu_mn->refs);
300 return smmu_mn;
301 }
302 }
303
304 cd = arm_smmu_alloc_shared_cd(mm);
305 if (IS_ERR(ptr: cd))
306 return ERR_CAST(ptr: cd);
307
308 smmu_mn = kzalloc(size: sizeof(*smmu_mn), GFP_KERNEL);
309 if (!smmu_mn) {
310 ret = -ENOMEM;
311 goto err_free_cd;
312 }
313
314 refcount_set(r: &smmu_mn->refs, n: 1);
315 smmu_mn->cd = cd;
316 smmu_mn->domain = smmu_domain;
317 smmu_mn->mn.ops = &arm_smmu_mmu_notifier_ops;
318
319 ret = mmu_notifier_register(subscription: &smmu_mn->mn, mm);
320 if (ret) {
321 kfree(objp: smmu_mn);
322 goto err_free_cd;
323 }
324
325 list_add(new: &smmu_mn->list, head: &smmu_domain->mmu_notifiers);
326 return smmu_mn;
327
328err_free_cd:
329 arm_smmu_free_shared_cd(cd);
330 return ERR_PTR(error: ret);
331}
332
333static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
334{
335 struct mm_struct *mm = smmu_mn->mn.mm;
336 struct arm_smmu_ctx_desc *cd = smmu_mn->cd;
337 struct arm_smmu_domain *smmu_domain = smmu_mn->domain;
338
339 if (!refcount_dec_and_test(r: &smmu_mn->refs))
340 return;
341
342 list_del(entry: &smmu_mn->list);
343
344 /*
345 * If we went through clear(), we've already invalidated, and no
346 * new TLB entry can have been formed.
347 */
348 if (!smmu_mn->cleared) {
349 arm_smmu_tlb_inv_asid(smmu: smmu_domain->smmu, asid: cd->asid);
350 arm_smmu_atc_inv_domain(smmu_domain, ssid: mm_get_enqcmd_pasid(mm), iova: 0,
351 size: 0);
352 }
353
354 /* Frees smmu_mn */
355 mmu_notifier_put(subscription: &smmu_mn->mn);
356 arm_smmu_free_shared_cd(cd);
357}
358
359static int __arm_smmu_sva_bind(struct device *dev, ioasid_t pasid,
360 struct mm_struct *mm)
361{
362 int ret;
363 struct arm_smmu_bond *bond;
364 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
365 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
366 struct arm_smmu_domain *smmu_domain;
367
368 if (!(domain->type & __IOMMU_DOMAIN_PAGING))
369 return -ENODEV;
370 smmu_domain = to_smmu_domain(dom: domain);
371 if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
372 return -ENODEV;
373
374 if (!master || !master->sva_enabled)
375 return -ENODEV;
376
377 bond = kzalloc(size: sizeof(*bond), GFP_KERNEL);
378 if (!bond)
379 return -ENOMEM;
380
381 bond->mm = mm;
382
383 bond->smmu_mn = arm_smmu_mmu_notifier_get(smmu_domain, mm);
384 if (IS_ERR(ptr: bond->smmu_mn)) {
385 ret = PTR_ERR(ptr: bond->smmu_mn);
386 goto err_free_bond;
387 }
388
389 ret = arm_smmu_write_ctx_desc(smmu_master: master, ssid: pasid, cd: bond->smmu_mn->cd);
390 if (ret)
391 goto err_put_notifier;
392
393 list_add(new: &bond->list, head: &master->bonds);
394 return 0;
395
396err_put_notifier:
397 arm_smmu_mmu_notifier_put(smmu_mn: bond->smmu_mn);
398err_free_bond:
399 kfree(objp: bond);
400 return ret;
401}
402
403bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
404{
405 unsigned long reg, fld;
406 unsigned long oas;
407 unsigned long asid_bits;
408 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
409
410 if (vabits_actual == 52)
411 feat_mask |= ARM_SMMU_FEAT_VAX;
412
413 if ((smmu->features & feat_mask) != feat_mask)
414 return false;
415
416 if (!(smmu->pgsize_bitmap & PAGE_SIZE))
417 return false;
418
419 /*
420 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're
421 * not even pretending to support AArch32 here. Abort if the MMU outputs
422 * addresses larger than what we support.
423 */
424 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
425 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
426 oas = id_aa64mmfr0_parange_to_phys_shift(fld);
427 if (smmu->oas < oas)
428 return false;
429
430 /* We can support bigger ASIDs than the CPU, but not smaller */
431 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
432 asid_bits = fld ? 16 : 8;
433 if (smmu->asid_bits < asid_bits)
434 return false;
435
436 /*
437 * See max_pinned_asids in arch/arm64/mm/context.c. The following is
438 * generally the maximum number of bindable processes.
439 */
440 if (arm64_kernel_unmapped_at_el0())
441 asid_bits--;
442 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) -
443 num_possible_cpus() - 2);
444
445 return true;
446}
447
448bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master)
449{
450 /* We're not keeping track of SIDs in fault events */
451 if (master->num_streams != 1)
452 return false;
453
454 return master->stall_enabled;
455}
456
457bool arm_smmu_master_sva_supported(struct arm_smmu_master *master)
458{
459 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
460 return false;
461
462 /* SSID support is mandatory for the moment */
463 return master->ssid_bits;
464}
465
466bool arm_smmu_master_sva_enabled(struct arm_smmu_master *master)
467{
468 bool enabled;
469
470 mutex_lock(&sva_lock);
471 enabled = master->sva_enabled;
472 mutex_unlock(lock: &sva_lock);
473 return enabled;
474}
475
476static int arm_smmu_master_sva_enable_iopf(struct arm_smmu_master *master)
477{
478 struct device *dev = master->dev;
479
480 /*
481 * Drivers for devices supporting PRI or stall should enable IOPF first.
482 * Others have device-specific fault handlers and don't need IOPF.
483 */
484 if (!arm_smmu_master_iopf_supported(master))
485 return 0;
486
487 if (!master->iopf_enabled)
488 return -EINVAL;
489
490 return iopf_queue_add_device(queue: master->smmu->evtq.iopf, dev);
491}
492
493static void arm_smmu_master_sva_disable_iopf(struct arm_smmu_master *master)
494{
495 struct device *dev = master->dev;
496
497 if (!master->iopf_enabled)
498 return;
499
500 iopf_queue_remove_device(queue: master->smmu->evtq.iopf, dev);
501}
502
503int arm_smmu_master_enable_sva(struct arm_smmu_master *master)
504{
505 int ret;
506
507 mutex_lock(&sva_lock);
508 ret = arm_smmu_master_sva_enable_iopf(master);
509 if (!ret)
510 master->sva_enabled = true;
511 mutex_unlock(lock: &sva_lock);
512
513 return ret;
514}
515
516int arm_smmu_master_disable_sva(struct arm_smmu_master *master)
517{
518 mutex_lock(&sva_lock);
519 if (!list_empty(head: &master->bonds)) {
520 dev_err(master->dev, "cannot disable SVA, device is bound\n");
521 mutex_unlock(lock: &sva_lock);
522 return -EBUSY;
523 }
524 arm_smmu_master_sva_disable_iopf(master);
525 master->sva_enabled = false;
526 mutex_unlock(lock: &sva_lock);
527
528 return 0;
529}
530
531void arm_smmu_sva_notifier_synchronize(void)
532{
533 /*
534 * Some MMU notifiers may still be waiting to be freed, using
535 * arm_smmu_mmu_notifier_free(). Wait for them.
536 */
537 mmu_notifier_synchronize();
538}
539
540void arm_smmu_sva_remove_dev_pasid(struct iommu_domain *domain,
541 struct device *dev, ioasid_t id)
542{
543 struct mm_struct *mm = domain->mm;
544 struct arm_smmu_bond *bond = NULL, *t;
545 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
546
547 mutex_lock(&sva_lock);
548
549 arm_smmu_write_ctx_desc(smmu_master: master, ssid: id, NULL);
550
551 list_for_each_entry(t, &master->bonds, list) {
552 if (t->mm == mm) {
553 bond = t;
554 break;
555 }
556 }
557
558 if (!WARN_ON(!bond)) {
559 list_del(entry: &bond->list);
560 arm_smmu_mmu_notifier_put(smmu_mn: bond->smmu_mn);
561 kfree(objp: bond);
562 }
563 mutex_unlock(lock: &sva_lock);
564}
565
566static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
567 struct device *dev, ioasid_t id)
568{
569 int ret = 0;
570 struct mm_struct *mm = domain->mm;
571
572 mutex_lock(&sva_lock);
573 ret = __arm_smmu_sva_bind(dev, pasid: id, mm);
574 mutex_unlock(lock: &sva_lock);
575
576 return ret;
577}
578
579static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
580{
581 kfree(objp: domain);
582}
583
584static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
585 .set_dev_pasid = arm_smmu_sva_set_dev_pasid,
586 .free = arm_smmu_sva_domain_free
587};
588
589struct iommu_domain *arm_smmu_sva_domain_alloc(void)
590{
591 struct iommu_domain *domain;
592
593 domain = kzalloc(size: sizeof(*domain), GFP_KERNEL);
594 if (!domain)
595 return NULL;
596 domain->ops = &arm_smmu_sva_domain_ops;
597
598 return domain;
599}
600

source code of linux/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c