1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * iommu.c: IOMMU specific routines for memory management. |
4 | * |
5 | * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) |
6 | * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) |
7 | * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) |
8 | * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) |
9 | */ |
10 | |
11 | #include <linux/kernel.h> |
12 | #include <linux/init.h> |
13 | #include <linux/mm.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/dma-map-ops.h> |
16 | #include <linux/of.h> |
17 | #include <linux/of_platform.h> |
18 | #include <linux/platform_device.h> |
19 | |
20 | #include <asm/io.h> |
21 | #include <asm/mxcc.h> |
22 | #include <asm/mbus.h> |
23 | #include <asm/cacheflush.h> |
24 | #include <asm/tlbflush.h> |
25 | #include <asm/bitext.h> |
26 | #include <asm/iommu.h> |
27 | #include <asm/dma.h> |
28 | |
29 | #include "mm_32.h" |
30 | |
31 | /* |
32 | * This can be sized dynamically, but we will do this |
33 | * only when we have a guidance about actual I/O pressures. |
34 | */ |
35 | #define IOMMU_RNGE IOMMU_RNGE_256MB |
36 | #define IOMMU_START 0xF0000000 |
37 | #define IOMMU_WINSIZE (256*1024*1024U) |
38 | #define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ |
39 | #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ |
40 | |
41 | static int viking_flush; |
42 | /* viking.S */ |
43 | extern void viking_flush_page(unsigned long page); |
44 | extern void viking_mxcc_flush_page(unsigned long page); |
45 | |
46 | /* |
47 | * Values precomputed according to CPU type. |
48 | */ |
49 | static unsigned int ioperm_noc; /* Consistent mapping iopte flags */ |
50 | static pgprot_t dvma_prot; /* Consistent mapping pte flags */ |
51 | |
52 | #define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) |
53 | #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) |
54 | |
55 | static const struct dma_map_ops sbus_iommu_dma_gflush_ops; |
56 | static const struct dma_map_ops sbus_iommu_dma_pflush_ops; |
57 | |
58 | static void __init sbus_iommu_init(struct platform_device *op) |
59 | { |
60 | struct iommu_struct *iommu; |
61 | unsigned int impl, vers; |
62 | unsigned long *bitmap; |
63 | unsigned long control; |
64 | unsigned long base; |
65 | unsigned long tmp; |
66 | |
67 | iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); |
68 | if (!iommu) { |
69 | prom_printf("Unable to allocate iommu structure\n" ); |
70 | prom_halt(); |
71 | } |
72 | |
73 | iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3, |
74 | "iommu_regs" ); |
75 | if (!iommu->regs) { |
76 | prom_printf("Cannot map IOMMU registers\n" ); |
77 | prom_halt(); |
78 | } |
79 | |
80 | control = sbus_readl(&iommu->regs->control); |
81 | impl = (control & IOMMU_CTRL_IMPL) >> 28; |
82 | vers = (control & IOMMU_CTRL_VERS) >> 24; |
83 | control &= ~(IOMMU_CTRL_RNGE); |
84 | control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); |
85 | sbus_writel(control, &iommu->regs->control); |
86 | |
87 | iommu_invalidate(iommu->regs); |
88 | iommu->start = IOMMU_START; |
89 | iommu->end = 0xffffffff; |
90 | |
91 | /* Allocate IOMMU page table */ |
92 | /* Stupid alignment constraints give me a headache. |
93 | We need 256K or 512K or 1M or 2M area aligned to |
94 | its size and current gfp will fortunately give |
95 | it to us. */ |
96 | tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); |
97 | if (!tmp) { |
98 | prom_printf("Unable to allocate iommu table [0x%lx]\n" , |
99 | IOMMU_NPTES * sizeof(iopte_t)); |
100 | prom_halt(); |
101 | } |
102 | iommu->page_table = (iopte_t *)tmp; |
103 | |
104 | /* Initialize new table. */ |
105 | memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); |
106 | flush_cache_all(); |
107 | flush_tlb_all(); |
108 | |
109 | base = __pa((unsigned long)iommu->page_table) >> 4; |
110 | sbus_writel(base, &iommu->regs->base); |
111 | iommu_invalidate(iommu->regs); |
112 | |
113 | bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); |
114 | if (!bitmap) { |
115 | prom_printf("Unable to allocate iommu bitmap [%d]\n" , |
116 | (int)(IOMMU_NPTES>>3)); |
117 | prom_halt(); |
118 | } |
119 | bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES); |
120 | /* To be coherent on HyperSparc, the page color of DVMA |
121 | * and physical addresses must match. |
122 | */ |
123 | if (srmmu_modtype == HyperSparc) |
124 | iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT; |
125 | else |
126 | iommu->usemap.num_colors = 1; |
127 | |
128 | printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n" , |
129 | impl, vers, iommu->page_table, |
130 | (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); |
131 | |
132 | op->dev.archdata.iommu = iommu; |
133 | |
134 | if (flush_page_for_dma_global) |
135 | op->dev.dma_ops = &sbus_iommu_dma_gflush_ops; |
136 | else |
137 | op->dev.dma_ops = &sbus_iommu_dma_pflush_ops; |
138 | } |
139 | |
140 | static int __init iommu_init(void) |
141 | { |
142 | struct device_node *dp; |
143 | |
144 | for_each_node_by_name(dp, "iommu" ) { |
145 | struct platform_device *op = of_find_device_by_node(np: dp); |
146 | |
147 | sbus_iommu_init(op); |
148 | of_propagate_archdata(op); |
149 | } |
150 | |
151 | return 0; |
152 | } |
153 | |
154 | subsys_initcall(iommu_init); |
155 | |
156 | /* Flush the iotlb entries to ram. */ |
157 | /* This could be better if we didn't have to flush whole pages. */ |
158 | static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) |
159 | { |
160 | unsigned long start; |
161 | unsigned long end; |
162 | |
163 | start = (unsigned long)iopte; |
164 | end = PAGE_ALIGN(start + niopte*sizeof(iopte_t)); |
165 | start &= PAGE_MASK; |
166 | if (viking_mxcc_present) { |
167 | while(start < end) { |
168 | viking_mxcc_flush_page(page: start); |
169 | start += PAGE_SIZE; |
170 | } |
171 | } else if (viking_flush) { |
172 | while(start < end) { |
173 | viking_flush_page(page: start); |
174 | start += PAGE_SIZE; |
175 | } |
176 | } else { |
177 | while(start < end) { |
178 | __flush_page_to_ram(start); |
179 | start += PAGE_SIZE; |
180 | } |
181 | } |
182 | } |
183 | |
184 | static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page, |
185 | unsigned long offset, size_t len, bool per_page_flush) |
186 | { |
187 | struct iommu_struct *iommu = dev->archdata.iommu; |
188 | phys_addr_t paddr = page_to_phys(page) + offset; |
189 | unsigned long off = paddr & ~PAGE_MASK; |
190 | unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; |
191 | unsigned long pfn = __phys_to_pfn(paddr); |
192 | unsigned int busa, busa0; |
193 | iopte_t *iopte, *iopte0; |
194 | int ioptex, i; |
195 | |
196 | /* XXX So what is maxphys for us and how do drivers know it? */ |
197 | if (!len || len > 256 * 1024) |
198 | return DMA_MAPPING_ERROR; |
199 | |
200 | /* |
201 | * We expect unmapped highmem pages to be not in the cache. |
202 | * XXX Is this a good assumption? |
203 | * XXX What if someone else unmaps it here and races us? |
204 | */ |
205 | if (per_page_flush && !PageHighMem(page)) { |
206 | unsigned long vaddr, p; |
207 | |
208 | vaddr = (unsigned long)page_address(page) + offset; |
209 | for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE) |
210 | flush_page_for_dma(p); |
211 | } |
212 | |
213 | /* page color = pfn of page */ |
214 | ioptex = bit_map_string_get(&iommu->usemap, npages, pfn); |
215 | if (ioptex < 0) |
216 | panic(fmt: "iommu out" ); |
217 | busa0 = iommu->start + (ioptex << PAGE_SHIFT); |
218 | iopte0 = &iommu->page_table[ioptex]; |
219 | |
220 | busa = busa0; |
221 | iopte = iopte0; |
222 | for (i = 0; i < npages; i++) { |
223 | iopte_val(*iopte) = MKIOPTE(pfn, IOPERM); |
224 | iommu_invalidate_page(iommu->regs, busa); |
225 | busa += PAGE_SIZE; |
226 | iopte++; |
227 | pfn++; |
228 | } |
229 | |
230 | iommu_flush_iotlb(iopte: iopte0, niopte: npages); |
231 | return busa0 + off; |
232 | } |
233 | |
234 | static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev, |
235 | struct page *page, unsigned long offset, size_t len, |
236 | enum dma_data_direction dir, unsigned long attrs) |
237 | { |
238 | flush_page_for_dma(0); |
239 | return __sbus_iommu_map_page(dev, page, offset, len, per_page_flush: false); |
240 | } |
241 | |
242 | static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev, |
243 | struct page *page, unsigned long offset, size_t len, |
244 | enum dma_data_direction dir, unsigned long attrs) |
245 | { |
246 | return __sbus_iommu_map_page(dev, page, offset, len, per_page_flush: true); |
247 | } |
248 | |
249 | static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, |
250 | int nents, enum dma_data_direction dir, unsigned long attrs, |
251 | bool per_page_flush) |
252 | { |
253 | struct scatterlist *sg; |
254 | int j; |
255 | |
256 | for_each_sg(sgl, sg, nents, j) { |
257 | sg->dma_address =__sbus_iommu_map_page(dev, page: sg_page(sg), |
258 | offset: sg->offset, len: sg->length, per_page_flush); |
259 | if (sg->dma_address == DMA_MAPPING_ERROR) |
260 | return -EIO; |
261 | sg->dma_length = sg->length; |
262 | } |
263 | |
264 | return nents; |
265 | } |
266 | |
267 | static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl, |
268 | int nents, enum dma_data_direction dir, unsigned long attrs) |
269 | { |
270 | flush_page_for_dma(0); |
271 | return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, per_page_flush: false); |
272 | } |
273 | |
274 | static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl, |
275 | int nents, enum dma_data_direction dir, unsigned long attrs) |
276 | { |
277 | return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, per_page_flush: true); |
278 | } |
279 | |
280 | static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr, |
281 | size_t len, enum dma_data_direction dir, unsigned long attrs) |
282 | { |
283 | struct iommu_struct *iommu = dev->archdata.iommu; |
284 | unsigned int busa = dma_addr & PAGE_MASK; |
285 | unsigned long off = dma_addr & ~PAGE_MASK; |
286 | unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; |
287 | unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT; |
288 | unsigned int i; |
289 | |
290 | BUG_ON(busa < iommu->start); |
291 | for (i = 0; i < npages; i++) { |
292 | iopte_val(iommu->page_table[ioptex + i]) = 0; |
293 | iommu_invalidate_page(iommu->regs, busa); |
294 | busa += PAGE_SIZE; |
295 | } |
296 | bit_map_clear(&iommu->usemap, ioptex, npages); |
297 | } |
298 | |
299 | static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, |
300 | int nents, enum dma_data_direction dir, unsigned long attrs) |
301 | { |
302 | struct scatterlist *sg; |
303 | int i; |
304 | |
305 | for_each_sg(sgl, sg, nents, i) { |
306 | sbus_iommu_unmap_page(dev, dma_addr: sg->dma_address, len: sg->length, dir, |
307 | attrs); |
308 | sg->dma_address = 0x21212121; |
309 | } |
310 | } |
311 | |
312 | #ifdef CONFIG_SBUS |
313 | static void *sbus_iommu_alloc(struct device *dev, size_t len, |
314 | dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) |
315 | { |
316 | struct iommu_struct *iommu = dev->archdata.iommu; |
317 | unsigned long va, addr, page, end, ret; |
318 | iopte_t *iopte = iommu->page_table; |
319 | iopte_t *first; |
320 | int ioptex; |
321 | |
322 | /* XXX So what is maxphys for us and how do drivers know it? */ |
323 | if (!len || len > 256 * 1024) |
324 | return NULL; |
325 | |
326 | len = PAGE_ALIGN(len); |
327 | va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); |
328 | if (va == 0) |
329 | return NULL; |
330 | |
331 | addr = ret = sparc_dma_alloc_resource(dev, len); |
332 | if (!addr) |
333 | goto out_free_pages; |
334 | |
335 | BUG_ON((va & ~PAGE_MASK) != 0); |
336 | BUG_ON((addr & ~PAGE_MASK) != 0); |
337 | BUG_ON((len & ~PAGE_MASK) != 0); |
338 | |
339 | /* page color = physical address */ |
340 | ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT, |
341 | addr >> PAGE_SHIFT); |
342 | if (ioptex < 0) |
343 | panic("iommu out" ); |
344 | |
345 | iopte += ioptex; |
346 | first = iopte; |
347 | end = addr + len; |
348 | while(addr < end) { |
349 | page = va; |
350 | { |
351 | pmd_t *pmdp; |
352 | pte_t *ptep; |
353 | |
354 | if (viking_mxcc_present) |
355 | viking_mxcc_flush_page(page); |
356 | else if (viking_flush) |
357 | viking_flush_page(page); |
358 | else |
359 | __flush_page_to_ram(page); |
360 | |
361 | pmdp = pmd_off_k(addr); |
362 | ptep = pte_offset_kernel(pmdp, addr); |
363 | |
364 | set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); |
365 | } |
366 | iopte_val(*iopte++) = |
367 | MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc); |
368 | addr += PAGE_SIZE; |
369 | va += PAGE_SIZE; |
370 | } |
371 | /* P3: why do we need this? |
372 | * |
373 | * DAVEM: Because there are several aspects, none of which |
374 | * are handled by a single interface. Some cpus are |
375 | * completely not I/O DMA coherent, and some have |
376 | * virtually indexed caches. The driver DMA flushing |
377 | * methods handle the former case, but here during |
378 | * IOMMU page table modifications, and usage of non-cacheable |
379 | * cpu mappings of pages potentially in the cpu caches, we have |
380 | * to handle the latter case as well. |
381 | */ |
382 | flush_cache_all(); |
383 | iommu_flush_iotlb(first, len >> PAGE_SHIFT); |
384 | flush_tlb_all(); |
385 | iommu_invalidate(iommu->regs); |
386 | |
387 | *dma_handle = iommu->start + (ioptex << PAGE_SHIFT); |
388 | return (void *)ret; |
389 | |
390 | out_free_pages: |
391 | free_pages(va, get_order(len)); |
392 | return NULL; |
393 | } |
394 | |
395 | static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr, |
396 | dma_addr_t busa, unsigned long attrs) |
397 | { |
398 | struct iommu_struct *iommu = dev->archdata.iommu; |
399 | iopte_t *iopte = iommu->page_table; |
400 | struct page *page = virt_to_page(cpu_addr); |
401 | int ioptex = (busa - iommu->start) >> PAGE_SHIFT; |
402 | unsigned long end; |
403 | |
404 | if (!sparc_dma_free_resource(cpu_addr, len)) |
405 | return; |
406 | |
407 | BUG_ON((busa & ~PAGE_MASK) != 0); |
408 | BUG_ON((len & ~PAGE_MASK) != 0); |
409 | |
410 | iopte += ioptex; |
411 | end = busa + len; |
412 | while (busa < end) { |
413 | iopte_val(*iopte++) = 0; |
414 | busa += PAGE_SIZE; |
415 | } |
416 | flush_tlb_all(); |
417 | iommu_invalidate(iommu->regs); |
418 | bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); |
419 | |
420 | __free_pages(page, get_order(len)); |
421 | } |
422 | #endif |
423 | |
424 | static const struct dma_map_ops sbus_iommu_dma_gflush_ops = { |
425 | #ifdef CONFIG_SBUS |
426 | .alloc = sbus_iommu_alloc, |
427 | .free = sbus_iommu_free, |
428 | #endif |
429 | .map_page = sbus_iommu_map_page_gflush, |
430 | .unmap_page = sbus_iommu_unmap_page, |
431 | .map_sg = sbus_iommu_map_sg_gflush, |
432 | .unmap_sg = sbus_iommu_unmap_sg, |
433 | }; |
434 | |
435 | static const struct dma_map_ops sbus_iommu_dma_pflush_ops = { |
436 | #ifdef CONFIG_SBUS |
437 | .alloc = sbus_iommu_alloc, |
438 | .free = sbus_iommu_free, |
439 | #endif |
440 | .map_page = sbus_iommu_map_page_pflush, |
441 | .unmap_page = sbus_iommu_unmap_page, |
442 | .map_sg = sbus_iommu_map_sg_pflush, |
443 | .unmap_sg = sbus_iommu_unmap_sg, |
444 | }; |
445 | |
446 | void __init ld_mmu_iommu(void) |
447 | { |
448 | if (viking_mxcc_present || srmmu_modtype == HyperSparc) { |
449 | dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); |
450 | ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; |
451 | } else { |
452 | dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV); |
453 | ioperm_noc = IOPTE_WRITE | IOPTE_VALID; |
454 | } |
455 | } |
456 | |