1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * IOMMU API for Renesas VMSA-compatible IPMMU |
4 | * Author: Laurent Pinchart <laurent.pinchart@ideasonboard.com> |
5 | * |
6 | * Copyright (C) 2014-2020 Renesas Electronics Corporation |
7 | */ |
8 | |
9 | #include <linux/bitmap.h> |
10 | #include <linux/delay.h> |
11 | #include <linux/dma-mapping.h> |
12 | #include <linux/err.h> |
13 | #include <linux/export.h> |
14 | #include <linux/init.h> |
15 | #include <linux/interrupt.h> |
16 | #include <linux/io.h> |
17 | #include <linux/iopoll.h> |
18 | #include <linux/io-pgtable.h> |
19 | #include <linux/iommu.h> |
20 | #include <linux/of.h> |
21 | #include <linux/of_platform.h> |
22 | #include <linux/pci.h> |
23 | #include <linux/platform_device.h> |
24 | #include <linux/sizes.h> |
25 | #include <linux/slab.h> |
26 | #include <linux/sys_soc.h> |
27 | |
28 | #if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) |
29 | #include <asm/dma-iommu.h> |
30 | #else |
31 | #define arm_iommu_create_mapping(...) NULL |
32 | #define arm_iommu_attach_device(...) -ENODEV |
33 | #define arm_iommu_release_mapping(...) do {} while (0) |
34 | #endif |
35 | |
36 | #define IPMMU_CTX_MAX 16U |
37 | #define IPMMU_CTX_INVALID -1 |
38 | |
39 | #define IPMMU_UTLB_MAX 64U |
40 | |
41 | struct ipmmu_features { |
42 | bool use_ns_alias_offset; |
43 | bool has_cache_leaf_nodes; |
44 | unsigned int number_of_contexts; |
45 | unsigned int num_utlbs; |
46 | bool setup_imbuscr; |
47 | bool twobit_imttbcr_sl0; |
48 | bool reserved_context; |
49 | bool cache_snoop; |
50 | unsigned int ctx_offset_base; |
51 | unsigned int ctx_offset_stride; |
52 | unsigned int utlb_offset_base; |
53 | }; |
54 | |
55 | struct ipmmu_vmsa_device { |
56 | struct device *dev; |
57 | void __iomem *base; |
58 | struct iommu_device iommu; |
59 | struct ipmmu_vmsa_device *root; |
60 | const struct ipmmu_features *features; |
61 | unsigned int num_ctx; |
62 | spinlock_t lock; /* Protects ctx and domains[] */ |
63 | DECLARE_BITMAP(ctx, IPMMU_CTX_MAX); |
64 | struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX]; |
65 | s8 utlb_ctx[IPMMU_UTLB_MAX]; |
66 | |
67 | struct dma_iommu_mapping *mapping; |
68 | }; |
69 | |
70 | struct ipmmu_vmsa_domain { |
71 | struct ipmmu_vmsa_device *mmu; |
72 | struct iommu_domain io_domain; |
73 | |
74 | struct io_pgtable_cfg cfg; |
75 | struct io_pgtable_ops *iop; |
76 | |
77 | unsigned int context_id; |
78 | struct mutex mutex; /* Protects mappings */ |
79 | }; |
80 | |
81 | static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) |
82 | { |
83 | return container_of(dom, struct ipmmu_vmsa_domain, io_domain); |
84 | } |
85 | |
86 | static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) |
87 | { |
88 | return dev_iommu_priv_get(dev); |
89 | } |
90 | |
91 | #define TLB_LOOP_TIMEOUT 100 /* 100us */ |
92 | |
93 | /* ----------------------------------------------------------------------------- |
94 | * Registers Definition |
95 | */ |
96 | |
97 | #define IM_NS_ALIAS_OFFSET 0x800 |
98 | |
99 | /* MMU "context" registers */ |
100 | #define IMCTR 0x0000 /* R-Car Gen2/3 */ |
101 | #define IMCTR_INTEN (1 << 2) /* R-Car Gen2/3 */ |
102 | #define IMCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */ |
103 | #define IMCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */ |
104 | |
105 | #define IMTTBCR 0x0008 /* R-Car Gen2/3 */ |
106 | #define IMTTBCR_EAE (1 << 31) /* R-Car Gen2/3 */ |
107 | #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */ |
108 | #define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */ |
109 | #define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */ |
110 | #define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */ |
111 | #define IMTTBCR_SL0_LVL_1 (1 << 4) /* R-Car Gen2 only */ |
112 | |
113 | #define IMBUSCR 0x000c /* R-Car Gen2 only */ |
114 | #define IMBUSCR_DVM (1 << 2) /* R-Car Gen2 only */ |
115 | #define IMBUSCR_BUSSEL_MASK (3 << 0) /* R-Car Gen2 only */ |
116 | |
117 | #define IMTTLBR0 0x0010 /* R-Car Gen2/3 */ |
118 | #define IMTTUBR0 0x0014 /* R-Car Gen2/3 */ |
119 | |
120 | #define IMSTR 0x0020 /* R-Car Gen2/3 */ |
121 | #define IMSTR_MHIT (1 << 4) /* R-Car Gen2/3 */ |
122 | #define IMSTR_ABORT (1 << 2) /* R-Car Gen2/3 */ |
123 | #define IMSTR_PF (1 << 1) /* R-Car Gen2/3 */ |
124 | #define IMSTR_TF (1 << 0) /* R-Car Gen2/3 */ |
125 | |
126 | #define IMMAIR0 0x0028 /* R-Car Gen2/3 */ |
127 | |
128 | #define IMELAR 0x0030 /* R-Car Gen2/3, IMEAR on R-Car Gen2 */ |
129 | #define IMEUAR 0x0034 /* R-Car Gen3 only */ |
130 | |
131 | /* uTLB registers */ |
132 | #define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n)) |
133 | #define IMUCTR0(n) (0x0300 + ((n) * 16)) /* R-Car Gen2/3 */ |
134 | #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) /* R-Car Gen3 only */ |
135 | #define IMUCTR_TTSEL_MMU(n) ((n) << 4) /* R-Car Gen2/3 */ |
136 | #define IMUCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */ |
137 | #define IMUCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */ |
138 | |
139 | #define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n)) |
140 | #define IMUASID0(n) (0x0308 + ((n) * 16)) /* R-Car Gen2/3 */ |
141 | #define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) /* R-Car Gen3 only */ |
142 | |
143 | /* ----------------------------------------------------------------------------- |
144 | * Root device handling |
145 | */ |
146 | |
147 | static struct platform_driver ipmmu_driver; |
148 | |
149 | static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu) |
150 | { |
151 | return mmu->root == mmu; |
152 | } |
153 | |
154 | static int __ipmmu_check_device(struct device *dev, void *data) |
155 | { |
156 | struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); |
157 | struct ipmmu_vmsa_device **rootp = data; |
158 | |
159 | if (ipmmu_is_root(mmu)) |
160 | *rootp = mmu; |
161 | |
162 | return 0; |
163 | } |
164 | |
165 | static struct ipmmu_vmsa_device *ipmmu_find_root(void) |
166 | { |
167 | struct ipmmu_vmsa_device *root = NULL; |
168 | |
169 | return driver_for_each_device(drv: &ipmmu_driver.driver, NULL, data: &root, |
170 | fn: __ipmmu_check_device) == 0 ? root : NULL; |
171 | } |
172 | |
173 | /* ----------------------------------------------------------------------------- |
174 | * Read/Write Access |
175 | */ |
176 | |
177 | static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset) |
178 | { |
179 | return ioread32(mmu->base + offset); |
180 | } |
181 | |
182 | static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, |
183 | u32 data) |
184 | { |
185 | iowrite32(data, mmu->base + offset); |
186 | } |
187 | |
188 | static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu, |
189 | unsigned int context_id, unsigned int reg) |
190 | { |
191 | unsigned int base = mmu->features->ctx_offset_base; |
192 | |
193 | if (context_id > 7) |
194 | base += 0x800 - 8 * 0x40; |
195 | |
196 | return base + context_id * mmu->features->ctx_offset_stride + reg; |
197 | } |
198 | |
199 | static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu, |
200 | unsigned int context_id, unsigned int reg) |
201 | { |
202 | return ipmmu_read(mmu, offset: ipmmu_ctx_reg(mmu, context_id, reg)); |
203 | } |
204 | |
205 | static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu, |
206 | unsigned int context_id, unsigned int reg, u32 data) |
207 | { |
208 | ipmmu_write(mmu, offset: ipmmu_ctx_reg(mmu, context_id, reg), data); |
209 | } |
210 | |
211 | static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain, |
212 | unsigned int reg) |
213 | { |
214 | return ipmmu_ctx_read(mmu: domain->mmu->root, context_id: domain->context_id, reg); |
215 | } |
216 | |
217 | static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain, |
218 | unsigned int reg, u32 data) |
219 | { |
220 | ipmmu_ctx_write(mmu: domain->mmu->root, context_id: domain->context_id, reg, data); |
221 | } |
222 | |
223 | static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain, |
224 | unsigned int reg, u32 data) |
225 | { |
226 | if (domain->mmu != domain->mmu->root) |
227 | ipmmu_ctx_write(mmu: domain->mmu, context_id: domain->context_id, reg, data); |
228 | |
229 | ipmmu_ctx_write(mmu: domain->mmu->root, context_id: domain->context_id, reg, data); |
230 | } |
231 | |
232 | static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg) |
233 | { |
234 | return mmu->features->utlb_offset_base + reg; |
235 | } |
236 | |
237 | static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu, |
238 | unsigned int utlb, u32 data) |
239 | { |
240 | ipmmu_write(mmu, offset: ipmmu_utlb_reg(mmu, IMUASID(utlb)), data); |
241 | } |
242 | |
243 | static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu, |
244 | unsigned int utlb, u32 data) |
245 | { |
246 | ipmmu_write(mmu, offset: ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data); |
247 | } |
248 | |
249 | /* ----------------------------------------------------------------------------- |
250 | * TLB and microTLB Management |
251 | */ |
252 | |
253 | /* Wait for any pending TLB invalidations to complete */ |
254 | static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) |
255 | { |
256 | u32 val; |
257 | |
258 | if (read_poll_timeout_atomic(ipmmu_ctx_read_root, val, |
259 | !(val & IMCTR_FLUSH), 1, TLB_LOOP_TIMEOUT, |
260 | false, domain, IMCTR)) |
261 | dev_err_ratelimited(domain->mmu->dev, |
262 | "TLB sync timed out -- MMU may be deadlocked\n" ); |
263 | } |
264 | |
265 | static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) |
266 | { |
267 | u32 reg; |
268 | |
269 | reg = ipmmu_ctx_read_root(domain, IMCTR); |
270 | reg |= IMCTR_FLUSH; |
271 | ipmmu_ctx_write_all(domain, IMCTR, data: reg); |
272 | |
273 | ipmmu_tlb_sync(domain); |
274 | } |
275 | |
276 | /* |
277 | * Enable MMU translation for the microTLB. |
278 | */ |
279 | static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, |
280 | unsigned int utlb) |
281 | { |
282 | struct ipmmu_vmsa_device *mmu = domain->mmu; |
283 | |
284 | /* |
285 | * TODO: Reference-count the microTLB as several bus masters can be |
286 | * connected to the same microTLB. |
287 | */ |
288 | |
289 | /* TODO: What should we set the ASID to ? */ |
290 | ipmmu_imuasid_write(mmu, utlb, data: 0); |
291 | /* TODO: Do we need to flush the microTLB ? */ |
292 | ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) | |
293 | IMUCTR_FLUSH | IMUCTR_MMUEN); |
294 | mmu->utlb_ctx[utlb] = domain->context_id; |
295 | } |
296 | |
297 | /* |
298 | * Disable MMU translation for the microTLB. |
299 | */ |
300 | static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, |
301 | unsigned int utlb) |
302 | { |
303 | struct ipmmu_vmsa_device *mmu = domain->mmu; |
304 | |
305 | ipmmu_imuctr_write(mmu, utlb, data: 0); |
306 | mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; |
307 | } |
308 | |
309 | static void ipmmu_tlb_flush_all(void *cookie) |
310 | { |
311 | struct ipmmu_vmsa_domain *domain = cookie; |
312 | |
313 | ipmmu_tlb_invalidate(domain); |
314 | } |
315 | |
316 | static void ipmmu_tlb_flush(unsigned long iova, size_t size, |
317 | size_t granule, void *cookie) |
318 | { |
319 | ipmmu_tlb_flush_all(cookie); |
320 | } |
321 | |
322 | static const struct iommu_flush_ops ipmmu_flush_ops = { |
323 | .tlb_flush_all = ipmmu_tlb_flush_all, |
324 | .tlb_flush_walk = ipmmu_tlb_flush, |
325 | }; |
326 | |
327 | /* ----------------------------------------------------------------------------- |
328 | * Domain/Context Management |
329 | */ |
330 | |
331 | static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu, |
332 | struct ipmmu_vmsa_domain *domain) |
333 | { |
334 | unsigned long flags; |
335 | int ret; |
336 | |
337 | spin_lock_irqsave(&mmu->lock, flags); |
338 | |
339 | ret = find_first_zero_bit(addr: mmu->ctx, size: mmu->num_ctx); |
340 | if (ret != mmu->num_ctx) { |
341 | mmu->domains[ret] = domain; |
342 | set_bit(nr: ret, addr: mmu->ctx); |
343 | } else |
344 | ret = -EBUSY; |
345 | |
346 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
347 | |
348 | return ret; |
349 | } |
350 | |
351 | static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu, |
352 | unsigned int context_id) |
353 | { |
354 | unsigned long flags; |
355 | |
356 | spin_lock_irqsave(&mmu->lock, flags); |
357 | |
358 | clear_bit(nr: context_id, addr: mmu->ctx); |
359 | mmu->domains[context_id] = NULL; |
360 | |
361 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
362 | } |
363 | |
364 | static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain) |
365 | { |
366 | u64 ttbr; |
367 | u32 tmp; |
368 | |
369 | /* TTBR0 */ |
370 | ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr; |
371 | ipmmu_ctx_write_root(domain, IMTTLBR0, data: ttbr); |
372 | ipmmu_ctx_write_root(domain, IMTTUBR0, data: ttbr >> 32); |
373 | |
374 | /* |
375 | * TTBCR |
376 | * We use long descriptors and allocate the whole 32-bit VA space to |
377 | * TTBR0. |
378 | */ |
379 | if (domain->mmu->features->twobit_imttbcr_sl0) |
380 | tmp = IMTTBCR_SL0_TWOBIT_LVL_1; |
381 | else |
382 | tmp = IMTTBCR_SL0_LVL_1; |
383 | |
384 | if (domain->mmu->features->cache_snoop) |
385 | tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | |
386 | IMTTBCR_IRGN0_WB_WA; |
387 | |
388 | ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp); |
389 | |
390 | /* MAIR0 */ |
391 | ipmmu_ctx_write_root(domain, IMMAIR0, |
392 | data: domain->cfg.arm_lpae_s1_cfg.mair); |
393 | |
394 | /* IMBUSCR */ |
395 | if (domain->mmu->features->setup_imbuscr) |
396 | ipmmu_ctx_write_root(domain, IMBUSCR, |
397 | data: ipmmu_ctx_read_root(domain, IMBUSCR) & |
398 | ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); |
399 | |
400 | /* |
401 | * IMSTR |
402 | * Clear all interrupt flags. |
403 | */ |
404 | ipmmu_ctx_write_root(domain, IMSTR, data: ipmmu_ctx_read_root(domain, IMSTR)); |
405 | |
406 | /* |
407 | * IMCTR |
408 | * Enable the MMU and interrupt generation. The long-descriptor |
409 | * translation table format doesn't use TEX remapping. Don't enable AF |
410 | * software management as we have no use for it. Flush the TLB as |
411 | * required when modifying the context registers. |
412 | */ |
413 | ipmmu_ctx_write_all(domain, IMCTR, |
414 | IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); |
415 | } |
416 | |
417 | static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) |
418 | { |
419 | int ret; |
420 | |
421 | /* |
422 | * Allocate the page table operations. |
423 | * |
424 | * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory |
425 | * access, Long-descriptor format" that the NStable bit being set in a |
426 | * table descriptor will result in the NStable and NS bits of all child |
427 | * entries being ignored and considered as being set. The IPMMU seems |
428 | * not to comply with this, as it generates a secure access page fault |
429 | * if any of the NStable and NS bits isn't set when running in |
430 | * non-secure mode. |
431 | */ |
432 | domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; |
433 | domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; |
434 | domain->cfg.ias = 32; |
435 | domain->cfg.oas = 40; |
436 | domain->cfg.tlb = &ipmmu_flush_ops; |
437 | domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); |
438 | domain->io_domain.geometry.force_aperture = true; |
439 | /* |
440 | * TODO: Add support for coherent walk through CCI with DVM and remove |
441 | * cache handling. For now, delegate it to the io-pgtable code. |
442 | */ |
443 | domain->cfg.coherent_walk = false; |
444 | domain->cfg.iommu_dev = domain->mmu->root->dev; |
445 | |
446 | /* |
447 | * Find an unused context. |
448 | */ |
449 | ret = ipmmu_domain_allocate_context(mmu: domain->mmu->root, domain); |
450 | if (ret < 0) |
451 | return ret; |
452 | |
453 | domain->context_id = ret; |
454 | |
455 | domain->iop = alloc_io_pgtable_ops(fmt: ARM_32_LPAE_S1, cfg: &domain->cfg, |
456 | cookie: domain); |
457 | if (!domain->iop) { |
458 | ipmmu_domain_free_context(mmu: domain->mmu->root, |
459 | context_id: domain->context_id); |
460 | return -EINVAL; |
461 | } |
462 | |
463 | ipmmu_domain_setup_context(domain); |
464 | return 0; |
465 | } |
466 | |
467 | static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) |
468 | { |
469 | if (!domain->mmu) |
470 | return; |
471 | |
472 | /* |
473 | * Disable the context. Flush the TLB as required when modifying the |
474 | * context registers. |
475 | * |
476 | * TODO: Is TLB flush really needed ? |
477 | */ |
478 | ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH); |
479 | ipmmu_tlb_sync(domain); |
480 | ipmmu_domain_free_context(mmu: domain->mmu->root, context_id: domain->context_id); |
481 | } |
482 | |
483 | /* ----------------------------------------------------------------------------- |
484 | * Fault Handling |
485 | */ |
486 | |
487 | static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) |
488 | { |
489 | const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; |
490 | struct ipmmu_vmsa_device *mmu = domain->mmu; |
491 | unsigned long iova; |
492 | u32 status; |
493 | |
494 | status = ipmmu_ctx_read_root(domain, IMSTR); |
495 | if (!(status & err_mask)) |
496 | return IRQ_NONE; |
497 | |
498 | iova = ipmmu_ctx_read_root(domain, IMELAR); |
499 | if (IS_ENABLED(CONFIG_64BIT)) |
500 | iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32; |
501 | |
502 | /* |
503 | * Clear the error status flags. Unlike traditional interrupt flag |
504 | * registers that must be cleared by writing 1, this status register |
505 | * seems to require 0. The error address register must be read before, |
506 | * otherwise its value will be 0. |
507 | */ |
508 | ipmmu_ctx_write_root(domain, IMSTR, data: 0); |
509 | |
510 | /* Log fatal errors. */ |
511 | if (status & IMSTR_MHIT) |
512 | dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n" , |
513 | iova); |
514 | if (status & IMSTR_ABORT) |
515 | dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n" , |
516 | iova); |
517 | |
518 | if (!(status & (IMSTR_PF | IMSTR_TF))) |
519 | return IRQ_NONE; |
520 | |
521 | /* |
522 | * Try to handle page faults and translation faults. |
523 | * |
524 | * TODO: We need to look up the faulty device based on the I/O VA. Use |
525 | * the IOMMU device for now. |
526 | */ |
527 | if (!report_iommu_fault(domain: &domain->io_domain, dev: mmu->dev, iova, flags: 0)) |
528 | return IRQ_HANDLED; |
529 | |
530 | dev_err_ratelimited(mmu->dev, |
531 | "Unhandled fault: status 0x%08x iova 0x%lx\n" , |
532 | status, iova); |
533 | |
534 | return IRQ_HANDLED; |
535 | } |
536 | |
537 | static irqreturn_t ipmmu_irq(int irq, void *dev) |
538 | { |
539 | struct ipmmu_vmsa_device *mmu = dev; |
540 | irqreturn_t status = IRQ_NONE; |
541 | unsigned int i; |
542 | unsigned long flags; |
543 | |
544 | spin_lock_irqsave(&mmu->lock, flags); |
545 | |
546 | /* |
547 | * Check interrupts for all active contexts. |
548 | */ |
549 | for (i = 0; i < mmu->num_ctx; i++) { |
550 | if (!mmu->domains[i]) |
551 | continue; |
552 | if (ipmmu_domain_irq(domain: mmu->domains[i]) == IRQ_HANDLED) |
553 | status = IRQ_HANDLED; |
554 | } |
555 | |
556 | spin_unlock_irqrestore(lock: &mmu->lock, flags); |
557 | |
558 | return status; |
559 | } |
560 | |
561 | /* ----------------------------------------------------------------------------- |
562 | * IOMMU Operations |
563 | */ |
564 | |
565 | static struct iommu_domain *ipmmu_domain_alloc_paging(struct device *dev) |
566 | { |
567 | struct ipmmu_vmsa_domain *domain; |
568 | |
569 | domain = kzalloc(size: sizeof(*domain), GFP_KERNEL); |
570 | if (!domain) |
571 | return NULL; |
572 | |
573 | mutex_init(&domain->mutex); |
574 | |
575 | return &domain->io_domain; |
576 | } |
577 | |
578 | static void ipmmu_domain_free(struct iommu_domain *io_domain) |
579 | { |
580 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(dom: io_domain); |
581 | |
582 | /* |
583 | * Free the domain resources. We assume that all devices have already |
584 | * been detached. |
585 | */ |
586 | ipmmu_domain_destroy_context(domain); |
587 | free_io_pgtable_ops(ops: domain->iop); |
588 | kfree(objp: domain); |
589 | } |
590 | |
591 | static int ipmmu_attach_device(struct iommu_domain *io_domain, |
592 | struct device *dev) |
593 | { |
594 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
595 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
596 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(dom: io_domain); |
597 | unsigned int i; |
598 | int ret = 0; |
599 | |
600 | if (!mmu) { |
601 | dev_err(dev, "Cannot attach to IPMMU\n" ); |
602 | return -ENXIO; |
603 | } |
604 | |
605 | mutex_lock(&domain->mutex); |
606 | |
607 | if (!domain->mmu) { |
608 | /* The domain hasn't been used yet, initialize it. */ |
609 | domain->mmu = mmu; |
610 | ret = ipmmu_domain_init_context(domain); |
611 | if (ret < 0) { |
612 | dev_err(dev, "Unable to initialize IPMMU context\n" ); |
613 | domain->mmu = NULL; |
614 | } else { |
615 | dev_info(dev, "Using IPMMU context %u\n" , |
616 | domain->context_id); |
617 | } |
618 | } else if (domain->mmu != mmu) { |
619 | /* |
620 | * Something is wrong, we can't attach two devices using |
621 | * different IOMMUs to the same domain. |
622 | */ |
623 | ret = -EINVAL; |
624 | } else |
625 | dev_info(dev, "Reusing IPMMU context %u\n" , domain->context_id); |
626 | |
627 | mutex_unlock(lock: &domain->mutex); |
628 | |
629 | if (ret < 0) |
630 | return ret; |
631 | |
632 | for (i = 0; i < fwspec->num_ids; ++i) |
633 | ipmmu_utlb_enable(domain, utlb: fwspec->ids[i]); |
634 | |
635 | return 0; |
636 | } |
637 | |
638 | static int ipmmu_iommu_identity_attach(struct iommu_domain *identity_domain, |
639 | struct device *dev) |
640 | { |
641 | struct iommu_domain *io_domain = iommu_get_domain_for_dev(dev); |
642 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
643 | struct ipmmu_vmsa_domain *domain; |
644 | unsigned int i; |
645 | |
646 | if (io_domain == identity_domain || !io_domain) |
647 | return 0; |
648 | |
649 | domain = to_vmsa_domain(dom: io_domain); |
650 | for (i = 0; i < fwspec->num_ids; ++i) |
651 | ipmmu_utlb_disable(domain, utlb: fwspec->ids[i]); |
652 | |
653 | /* |
654 | * TODO: Optimize by disabling the context when no device is attached. |
655 | */ |
656 | return 0; |
657 | } |
658 | |
659 | static struct iommu_domain_ops ipmmu_iommu_identity_ops = { |
660 | .attach_dev = ipmmu_iommu_identity_attach, |
661 | }; |
662 | |
663 | static struct iommu_domain ipmmu_iommu_identity_domain = { |
664 | .type = IOMMU_DOMAIN_IDENTITY, |
665 | .ops = &ipmmu_iommu_identity_ops, |
666 | }; |
667 | |
668 | static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, |
669 | phys_addr_t paddr, size_t pgsize, size_t pgcount, |
670 | int prot, gfp_t gfp, size_t *mapped) |
671 | { |
672 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(dom: io_domain); |
673 | |
674 | return domain->iop->map_pages(domain->iop, iova, paddr, pgsize, pgcount, |
675 | prot, gfp, mapped); |
676 | } |
677 | |
678 | static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, |
679 | size_t pgsize, size_t pgcount, |
680 | struct iommu_iotlb_gather *gather) |
681 | { |
682 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(dom: io_domain); |
683 | |
684 | return domain->iop->unmap_pages(domain->iop, iova, pgsize, pgcount, gather); |
685 | } |
686 | |
687 | static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain) |
688 | { |
689 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(dom: io_domain); |
690 | |
691 | if (domain->mmu) |
692 | ipmmu_tlb_flush_all(cookie: domain); |
693 | } |
694 | |
695 | static void ipmmu_iotlb_sync(struct iommu_domain *io_domain, |
696 | struct iommu_iotlb_gather *gather) |
697 | { |
698 | ipmmu_flush_iotlb_all(io_domain); |
699 | } |
700 | |
701 | static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, |
702 | dma_addr_t iova) |
703 | { |
704 | struct ipmmu_vmsa_domain *domain = to_vmsa_domain(dom: io_domain); |
705 | |
706 | /* TODO: Is locking needed ? */ |
707 | |
708 | return domain->iop->iova_to_phys(domain->iop, iova); |
709 | } |
710 | |
711 | static int ipmmu_init_platform_device(struct device *dev, |
712 | const struct of_phandle_args *args) |
713 | { |
714 | struct platform_device *ipmmu_pdev; |
715 | |
716 | ipmmu_pdev = of_find_device_by_node(np: args->np); |
717 | if (!ipmmu_pdev) |
718 | return -ENODEV; |
719 | |
720 | dev_iommu_priv_set(dev, priv: platform_get_drvdata(pdev: ipmmu_pdev)); |
721 | |
722 | return 0; |
723 | } |
724 | |
725 | static const struct soc_device_attribute soc_needs_opt_in[] = { |
726 | { .family = "R-Car Gen3" , }, |
727 | { .family = "R-Car Gen4" , }, |
728 | { .family = "RZ/G2" , }, |
729 | { /* sentinel */ } |
730 | }; |
731 | |
732 | static const struct soc_device_attribute soc_denylist[] = { |
733 | { .soc_id = "r8a774a1" , }, |
734 | { .soc_id = "r8a7795" , .revision = "ES2.*" }, |
735 | { .soc_id = "r8a7796" , }, |
736 | { /* sentinel */ } |
737 | }; |
738 | |
739 | static const char * const devices_allowlist[] = { |
740 | "ee100000.mmc" , |
741 | "ee120000.mmc" , |
742 | "ee140000.mmc" , |
743 | "ee160000.mmc" |
744 | }; |
745 | |
746 | static bool ipmmu_device_is_allowed(struct device *dev) |
747 | { |
748 | unsigned int i; |
749 | |
750 | /* |
751 | * R-Car Gen3/4 and RZ/G2 use the allow list to opt-in devices. |
752 | * For Other SoCs, this returns true anyway. |
753 | */ |
754 | if (!soc_device_match(matches: soc_needs_opt_in)) |
755 | return true; |
756 | |
757 | /* Check whether this SoC can use the IPMMU correctly or not */ |
758 | if (soc_device_match(matches: soc_denylist)) |
759 | return false; |
760 | |
761 | /* Check whether this device is a PCI device */ |
762 | if (dev_is_pci(dev)) |
763 | return true; |
764 | |
765 | /* Check whether this device can work with the IPMMU */ |
766 | for (i = 0; i < ARRAY_SIZE(devices_allowlist); i++) { |
767 | if (!strcmp(dev_name(dev), devices_allowlist[i])) |
768 | return true; |
769 | } |
770 | |
771 | /* Otherwise, do not allow use of IPMMU */ |
772 | return false; |
773 | } |
774 | |
775 | static int ipmmu_of_xlate(struct device *dev, |
776 | const struct of_phandle_args *spec) |
777 | { |
778 | if (!ipmmu_device_is_allowed(dev)) |
779 | return -ENODEV; |
780 | |
781 | iommu_fwspec_add_ids(dev, ids: spec->args, num_ids: 1); |
782 | |
783 | /* Initialize once - xlate() will call multiple times */ |
784 | if (to_ipmmu(dev)) |
785 | return 0; |
786 | |
787 | return ipmmu_init_platform_device(dev, args: spec); |
788 | } |
789 | |
790 | static int ipmmu_init_arm_mapping(struct device *dev) |
791 | { |
792 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
793 | int ret; |
794 | |
795 | /* |
796 | * Create the ARM mapping, used by the ARM DMA mapping core to allocate |
797 | * VAs. This will allocate a corresponding IOMMU domain. |
798 | * |
799 | * TODO: |
800 | * - Create one mapping per context (TLB). |
801 | * - Make the mapping size configurable ? We currently use a 2GB mapping |
802 | * at a 1GB offset to ensure that NULL VAs will fault. |
803 | */ |
804 | if (!mmu->mapping) { |
805 | struct dma_iommu_mapping *mapping; |
806 | |
807 | mapping = arm_iommu_create_mapping(&platform_bus_type, |
808 | SZ_1G, SZ_2G); |
809 | if (IS_ERR(ptr: mapping)) { |
810 | dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n" ); |
811 | ret = PTR_ERR(ptr: mapping); |
812 | goto error; |
813 | } |
814 | |
815 | mmu->mapping = mapping; |
816 | } |
817 | |
818 | /* Attach the ARM VA mapping to the device. */ |
819 | ret = arm_iommu_attach_device(dev, mmu->mapping); |
820 | if (ret < 0) { |
821 | dev_err(dev, "Failed to attach device to VA mapping\n" ); |
822 | goto error; |
823 | } |
824 | |
825 | return 0; |
826 | |
827 | error: |
828 | if (mmu->mapping) |
829 | arm_iommu_release_mapping(mmu->mapping); |
830 | |
831 | return ret; |
832 | } |
833 | |
834 | static struct iommu_device *ipmmu_probe_device(struct device *dev) |
835 | { |
836 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
837 | |
838 | /* |
839 | * Only let through devices that have been verified in xlate() |
840 | */ |
841 | if (!mmu) |
842 | return ERR_PTR(error: -ENODEV); |
843 | |
844 | return &mmu->iommu; |
845 | } |
846 | |
847 | static void ipmmu_probe_finalize(struct device *dev) |
848 | { |
849 | int ret = 0; |
850 | |
851 | if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)) |
852 | ret = ipmmu_init_arm_mapping(dev); |
853 | |
854 | if (ret) |
855 | dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n" ); |
856 | } |
857 | |
858 | static void ipmmu_release_device(struct device *dev) |
859 | { |
860 | struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); |
861 | struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); |
862 | unsigned int i; |
863 | |
864 | for (i = 0; i < fwspec->num_ids; ++i) { |
865 | unsigned int utlb = fwspec->ids[i]; |
866 | |
867 | ipmmu_imuctr_write(mmu, utlb, data: 0); |
868 | mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; |
869 | } |
870 | |
871 | arm_iommu_release_mapping(mmu->mapping); |
872 | } |
873 | |
874 | static const struct iommu_ops ipmmu_ops = { |
875 | .identity_domain = &ipmmu_iommu_identity_domain, |
876 | .domain_alloc_paging = ipmmu_domain_alloc_paging, |
877 | .probe_device = ipmmu_probe_device, |
878 | .release_device = ipmmu_release_device, |
879 | .probe_finalize = ipmmu_probe_finalize, |
880 | /* |
881 | * FIXME: The device grouping is a fixed property of the hardware's |
882 | * ability to isolate and control DMA, it should not depend on kconfig. |
883 | */ |
884 | .device_group = IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA) |
885 | ? generic_device_group : generic_single_device_group, |
886 | .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, |
887 | .of_xlate = ipmmu_of_xlate, |
888 | .default_domain_ops = &(const struct iommu_domain_ops) { |
889 | .attach_dev = ipmmu_attach_device, |
890 | .map_pages = ipmmu_map, |
891 | .unmap_pages = ipmmu_unmap, |
892 | .flush_iotlb_all = ipmmu_flush_iotlb_all, |
893 | .iotlb_sync = ipmmu_iotlb_sync, |
894 | .iova_to_phys = ipmmu_iova_to_phys, |
895 | .free = ipmmu_domain_free, |
896 | } |
897 | }; |
898 | |
899 | /* ----------------------------------------------------------------------------- |
900 | * Probe/remove and init |
901 | */ |
902 | |
903 | static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) |
904 | { |
905 | unsigned int i; |
906 | |
907 | /* Disable all contexts. */ |
908 | for (i = 0; i < mmu->num_ctx; ++i) |
909 | ipmmu_ctx_write(mmu, context_id: i, IMCTR, data: 0); |
910 | } |
911 | |
912 | static const struct ipmmu_features ipmmu_features_default = { |
913 | .use_ns_alias_offset = true, |
914 | .has_cache_leaf_nodes = false, |
915 | .number_of_contexts = 1, /* software only tested with one context */ |
916 | .num_utlbs = 32, |
917 | .setup_imbuscr = true, |
918 | .twobit_imttbcr_sl0 = false, |
919 | .reserved_context = false, |
920 | .cache_snoop = true, |
921 | .ctx_offset_base = 0, |
922 | .ctx_offset_stride = 0x40, |
923 | .utlb_offset_base = 0, |
924 | }; |
925 | |
926 | static const struct ipmmu_features ipmmu_features_rcar_gen3 = { |
927 | .use_ns_alias_offset = false, |
928 | .has_cache_leaf_nodes = true, |
929 | .number_of_contexts = 8, |
930 | .num_utlbs = 48, |
931 | .setup_imbuscr = false, |
932 | .twobit_imttbcr_sl0 = true, |
933 | .reserved_context = true, |
934 | .cache_snoop = false, |
935 | .ctx_offset_base = 0, |
936 | .ctx_offset_stride = 0x40, |
937 | .utlb_offset_base = 0, |
938 | }; |
939 | |
940 | static const struct ipmmu_features ipmmu_features_rcar_gen4 = { |
941 | .use_ns_alias_offset = false, |
942 | .has_cache_leaf_nodes = true, |
943 | .number_of_contexts = 16, |
944 | .num_utlbs = 64, |
945 | .setup_imbuscr = false, |
946 | .twobit_imttbcr_sl0 = true, |
947 | .reserved_context = true, |
948 | .cache_snoop = false, |
949 | .ctx_offset_base = 0x10000, |
950 | .ctx_offset_stride = 0x1040, |
951 | .utlb_offset_base = 0x3000, |
952 | }; |
953 | |
954 | static const struct of_device_id ipmmu_of_ids[] = { |
955 | { |
956 | .compatible = "renesas,ipmmu-vmsa" , |
957 | .data = &ipmmu_features_default, |
958 | }, { |
959 | .compatible = "renesas,ipmmu-r8a774a1" , |
960 | .data = &ipmmu_features_rcar_gen3, |
961 | }, { |
962 | .compatible = "renesas,ipmmu-r8a774b1" , |
963 | .data = &ipmmu_features_rcar_gen3, |
964 | }, { |
965 | .compatible = "renesas,ipmmu-r8a774c0" , |
966 | .data = &ipmmu_features_rcar_gen3, |
967 | }, { |
968 | .compatible = "renesas,ipmmu-r8a774e1" , |
969 | .data = &ipmmu_features_rcar_gen3, |
970 | }, { |
971 | .compatible = "renesas,ipmmu-r8a7795" , |
972 | .data = &ipmmu_features_rcar_gen3, |
973 | }, { |
974 | .compatible = "renesas,ipmmu-r8a7796" , |
975 | .data = &ipmmu_features_rcar_gen3, |
976 | }, { |
977 | .compatible = "renesas,ipmmu-r8a77961" , |
978 | .data = &ipmmu_features_rcar_gen3, |
979 | }, { |
980 | .compatible = "renesas,ipmmu-r8a77965" , |
981 | .data = &ipmmu_features_rcar_gen3, |
982 | }, { |
983 | .compatible = "renesas,ipmmu-r8a77970" , |
984 | .data = &ipmmu_features_rcar_gen3, |
985 | }, { |
986 | .compatible = "renesas,ipmmu-r8a77980" , |
987 | .data = &ipmmu_features_rcar_gen3, |
988 | }, { |
989 | .compatible = "renesas,ipmmu-r8a77990" , |
990 | .data = &ipmmu_features_rcar_gen3, |
991 | }, { |
992 | .compatible = "renesas,ipmmu-r8a77995" , |
993 | .data = &ipmmu_features_rcar_gen3, |
994 | }, { |
995 | .compatible = "renesas,ipmmu-r8a779a0" , |
996 | .data = &ipmmu_features_rcar_gen4, |
997 | }, { |
998 | .compatible = "renesas,rcar-gen4-ipmmu-vmsa" , |
999 | .data = &ipmmu_features_rcar_gen4, |
1000 | }, { |
1001 | /* Terminator */ |
1002 | }, |
1003 | }; |
1004 | |
1005 | static int ipmmu_probe(struct platform_device *pdev) |
1006 | { |
1007 | struct ipmmu_vmsa_device *mmu; |
1008 | int irq; |
1009 | int ret; |
1010 | |
1011 | mmu = devm_kzalloc(dev: &pdev->dev, size: sizeof(*mmu), GFP_KERNEL); |
1012 | if (!mmu) { |
1013 | dev_err(&pdev->dev, "cannot allocate device data\n" ); |
1014 | return -ENOMEM; |
1015 | } |
1016 | |
1017 | mmu->dev = &pdev->dev; |
1018 | spin_lock_init(&mmu->lock); |
1019 | bitmap_zero(dst: mmu->ctx, IPMMU_CTX_MAX); |
1020 | mmu->features = of_device_get_match_data(dev: &pdev->dev); |
1021 | memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs); |
1022 | ret = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(40)); |
1023 | if (ret) |
1024 | return ret; |
1025 | |
1026 | /* Map I/O memory and request IRQ. */ |
1027 | mmu->base = devm_platform_ioremap_resource(pdev, index: 0); |
1028 | if (IS_ERR(ptr: mmu->base)) |
1029 | return PTR_ERR(ptr: mmu->base); |
1030 | |
1031 | /* |
1032 | * The IPMMU has two register banks, for secure and non-secure modes. |
1033 | * The bank mapped at the beginning of the IPMMU address space |
1034 | * corresponds to the running mode of the CPU. When running in secure |
1035 | * mode the non-secure register bank is also available at an offset. |
1036 | * |
1037 | * Secure mode operation isn't clearly documented and is thus currently |
1038 | * not implemented in the driver. Furthermore, preliminary tests of |
1039 | * non-secure operation with the main register bank were not successful. |
1040 | * Offset the registers base unconditionally to point to the non-secure |
1041 | * alias space for now. |
1042 | */ |
1043 | if (mmu->features->use_ns_alias_offset) |
1044 | mmu->base += IM_NS_ALIAS_OFFSET; |
1045 | |
1046 | mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts); |
1047 | |
1048 | /* |
1049 | * Determine if this IPMMU instance is a root device by checking for |
1050 | * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. |
1051 | */ |
1052 | if (!mmu->features->has_cache_leaf_nodes || |
1053 | !of_property_present(np: pdev->dev.of_node, propname: "renesas,ipmmu-main" )) |
1054 | mmu->root = mmu; |
1055 | else |
1056 | mmu->root = ipmmu_find_root(); |
1057 | |
1058 | /* |
1059 | * Wait until the root device has been registered for sure. |
1060 | */ |
1061 | if (!mmu->root) |
1062 | return -EPROBE_DEFER; |
1063 | |
1064 | /* Root devices have mandatory IRQs */ |
1065 | if (ipmmu_is_root(mmu)) { |
1066 | irq = platform_get_irq(pdev, 0); |
1067 | if (irq < 0) |
1068 | return irq; |
1069 | |
1070 | ret = devm_request_irq(dev: &pdev->dev, irq, handler: ipmmu_irq, irqflags: 0, |
1071 | devname: dev_name(dev: &pdev->dev), dev_id: mmu); |
1072 | if (ret < 0) { |
1073 | dev_err(&pdev->dev, "failed to request IRQ %d\n" , irq); |
1074 | return ret; |
1075 | } |
1076 | |
1077 | ipmmu_device_reset(mmu); |
1078 | |
1079 | if (mmu->features->reserved_context) { |
1080 | dev_info(&pdev->dev, "IPMMU context 0 is reserved\n" ); |
1081 | set_bit(nr: 0, addr: mmu->ctx); |
1082 | } |
1083 | } |
1084 | |
1085 | /* |
1086 | * Register the IPMMU to the IOMMU subsystem in the following cases: |
1087 | * - R-Car Gen2 IPMMU (all devices registered) |
1088 | * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) |
1089 | */ |
1090 | if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { |
1091 | ret = iommu_device_sysfs_add(iommu: &mmu->iommu, parent: &pdev->dev, NULL, |
1092 | fmt: dev_name(dev: &pdev->dev)); |
1093 | if (ret) |
1094 | return ret; |
1095 | |
1096 | ret = iommu_device_register(iommu: &mmu->iommu, ops: &ipmmu_ops, hwdev: &pdev->dev); |
1097 | if (ret) |
1098 | return ret; |
1099 | } |
1100 | |
1101 | /* |
1102 | * We can't create the ARM mapping here as it requires the bus to have |
1103 | * an IOMMU, which only happens when bus_set_iommu() is called in |
1104 | * ipmmu_init() after the probe function returns. |
1105 | */ |
1106 | |
1107 | platform_set_drvdata(pdev, data: mmu); |
1108 | |
1109 | return 0; |
1110 | } |
1111 | |
1112 | static void ipmmu_remove(struct platform_device *pdev) |
1113 | { |
1114 | struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); |
1115 | |
1116 | iommu_device_sysfs_remove(iommu: &mmu->iommu); |
1117 | iommu_device_unregister(iommu: &mmu->iommu); |
1118 | |
1119 | arm_iommu_release_mapping(mmu->mapping); |
1120 | |
1121 | ipmmu_device_reset(mmu); |
1122 | } |
1123 | |
1124 | static int ipmmu_resume_noirq(struct device *dev) |
1125 | { |
1126 | struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev); |
1127 | unsigned int i; |
1128 | |
1129 | /* Reset root MMU and restore contexts */ |
1130 | if (ipmmu_is_root(mmu)) { |
1131 | ipmmu_device_reset(mmu); |
1132 | |
1133 | for (i = 0; i < mmu->num_ctx; i++) { |
1134 | if (!mmu->domains[i]) |
1135 | continue; |
1136 | |
1137 | ipmmu_domain_setup_context(domain: mmu->domains[i]); |
1138 | } |
1139 | } |
1140 | |
1141 | /* Re-enable active micro-TLBs */ |
1142 | for (i = 0; i < mmu->features->num_utlbs; i++) { |
1143 | if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID) |
1144 | continue; |
1145 | |
1146 | ipmmu_utlb_enable(domain: mmu->root->domains[mmu->utlb_ctx[i]], utlb: i); |
1147 | } |
1148 | |
1149 | return 0; |
1150 | } |
1151 | |
1152 | static const struct dev_pm_ops ipmmu_pm = { |
1153 | NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq) |
1154 | }; |
1155 | |
1156 | static struct platform_driver ipmmu_driver = { |
1157 | .driver = { |
1158 | .name = "ipmmu-vmsa" , |
1159 | .of_match_table = ipmmu_of_ids, |
1160 | .pm = pm_sleep_ptr(&ipmmu_pm), |
1161 | }, |
1162 | .probe = ipmmu_probe, |
1163 | .remove_new = ipmmu_remove, |
1164 | }; |
1165 | builtin_platform_driver(ipmmu_driver); |
1166 | |