1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * CPU-agnostic AMD IO page table v2 allocator. |
4 | * |
5 | * Copyright (C) 2022, 2023 Advanced Micro Devices, Inc. |
6 | * Author: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> |
7 | * Author: Vasant Hegde <vasant.hegde@amd.com> |
8 | */ |
9 | |
10 | #define pr_fmt(fmt) "AMD-Vi: " fmt |
11 | #define dev_fmt(fmt) pr_fmt(fmt) |
12 | |
13 | #include <linux/bitops.h> |
14 | #include <linux/io-pgtable.h> |
15 | #include <linux/kernel.h> |
16 | |
17 | #include <asm/barrier.h> |
18 | |
19 | #include "amd_iommu_types.h" |
20 | #include "amd_iommu.h" |
21 | |
22 | #define IOMMU_PAGE_PRESENT BIT_ULL(0) /* Is present */ |
23 | #define IOMMU_PAGE_RW BIT_ULL(1) /* Writeable */ |
24 | #define IOMMU_PAGE_USER BIT_ULL(2) /* Userspace addressable */ |
25 | #define IOMMU_PAGE_PWT BIT_ULL(3) /* Page write through */ |
26 | #define IOMMU_PAGE_PCD BIT_ULL(4) /* Page cache disabled */ |
27 | #define IOMMU_PAGE_ACCESS BIT_ULL(5) /* Was accessed (updated by IOMMU) */ |
28 | #define IOMMU_PAGE_DIRTY BIT_ULL(6) /* Was written to (updated by IOMMU) */ |
29 | #define IOMMU_PAGE_PSE BIT_ULL(7) /* Page Size Extensions */ |
30 | #define IOMMU_PAGE_NX BIT_ULL(63) /* No execute */ |
31 | |
32 | #define MAX_PTRS_PER_PAGE 512 |
33 | |
34 | #define IOMMU_PAGE_SIZE_2M BIT_ULL(21) |
35 | #define IOMMU_PAGE_SIZE_1G BIT_ULL(30) |
36 | |
37 | |
38 | static inline int get_pgtable_level(void) |
39 | { |
40 | return amd_iommu_gpt_level; |
41 | } |
42 | |
43 | static inline bool is_large_pte(u64 pte) |
44 | { |
45 | return (pte & IOMMU_PAGE_PSE); |
46 | } |
47 | |
48 | static inline u64 set_pgtable_attr(u64 *page) |
49 | { |
50 | u64 prot; |
51 | |
52 | prot = IOMMU_PAGE_PRESENT | IOMMU_PAGE_RW | IOMMU_PAGE_USER; |
53 | prot |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY; |
54 | |
55 | return (iommu_virt_to_phys(vaddr: page) | prot); |
56 | } |
57 | |
58 | static inline void *get_pgtable_pte(u64 pte) |
59 | { |
60 | return iommu_phys_to_virt(paddr: pte & PM_ADDR_MASK); |
61 | } |
62 | |
63 | static u64 set_pte_attr(u64 paddr, u64 pg_size, int prot) |
64 | { |
65 | u64 pte; |
66 | |
67 | pte = __sme_set(paddr & PM_ADDR_MASK); |
68 | pte |= IOMMU_PAGE_PRESENT | IOMMU_PAGE_USER; |
69 | pte |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY; |
70 | |
71 | if (prot & IOMMU_PROT_IW) |
72 | pte |= IOMMU_PAGE_RW; |
73 | |
74 | /* Large page */ |
75 | if (pg_size == IOMMU_PAGE_SIZE_1G || pg_size == IOMMU_PAGE_SIZE_2M) |
76 | pte |= IOMMU_PAGE_PSE; |
77 | |
78 | return pte; |
79 | } |
80 | |
81 | static inline u64 get_alloc_page_size(u64 size) |
82 | { |
83 | if (size >= IOMMU_PAGE_SIZE_1G) |
84 | return IOMMU_PAGE_SIZE_1G; |
85 | |
86 | if (size >= IOMMU_PAGE_SIZE_2M) |
87 | return IOMMU_PAGE_SIZE_2M; |
88 | |
89 | return PAGE_SIZE; |
90 | } |
91 | |
92 | static inline int page_size_to_level(u64 pg_size) |
93 | { |
94 | if (pg_size == IOMMU_PAGE_SIZE_1G) |
95 | return PAGE_MODE_3_LEVEL; |
96 | if (pg_size == IOMMU_PAGE_SIZE_2M) |
97 | return PAGE_MODE_2_LEVEL; |
98 | |
99 | return PAGE_MODE_1_LEVEL; |
100 | } |
101 | |
102 | static inline void free_pgtable_page(u64 *pt) |
103 | { |
104 | free_page((unsigned long)pt); |
105 | } |
106 | |
107 | static void free_pgtable(u64 *pt, int level) |
108 | { |
109 | u64 *p; |
110 | int i; |
111 | |
112 | for (i = 0; i < MAX_PTRS_PER_PAGE; i++) { |
113 | /* PTE present? */ |
114 | if (!IOMMU_PTE_PRESENT(pt[i])) |
115 | continue; |
116 | |
117 | if (is_large_pte(pte: pt[i])) |
118 | continue; |
119 | |
120 | /* |
121 | * Free the next level. No need to look at l1 tables here since |
122 | * they can only contain leaf PTEs; just free them directly. |
123 | */ |
124 | p = get_pgtable_pte(pte: pt[i]); |
125 | if (level > 2) |
126 | free_pgtable(pt: p, level: level - 1); |
127 | else |
128 | free_pgtable_page(pt: p); |
129 | } |
130 | |
131 | free_pgtable_page(pt); |
132 | } |
133 | |
134 | /* Allocate page table */ |
135 | static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, |
136 | unsigned long pg_size, gfp_t gfp, bool *updated) |
137 | { |
138 | u64 *pte, *page; |
139 | int level, end_level; |
140 | |
141 | level = get_pgtable_level() - 1; |
142 | end_level = page_size_to_level(pg_size); |
143 | pte = &pgd[PM_LEVEL_INDEX(level, iova)]; |
144 | iova = PAGE_SIZE_ALIGN(iova, PAGE_SIZE); |
145 | |
146 | while (level >= end_level) { |
147 | u64 __pte, __npte; |
148 | |
149 | __pte = *pte; |
150 | |
151 | if (IOMMU_PTE_PRESENT(__pte) && is_large_pte(pte: __pte)) { |
152 | /* Unmap large pte */ |
153 | cmpxchg64(pte, *pte, 0ULL); |
154 | *updated = true; |
155 | continue; |
156 | } |
157 | |
158 | if (!IOMMU_PTE_PRESENT(__pte)) { |
159 | page = alloc_pgtable_page(nid, gfp); |
160 | if (!page) |
161 | return NULL; |
162 | |
163 | __npte = set_pgtable_attr(page); |
164 | /* pte could have been changed somewhere. */ |
165 | if (cmpxchg64(pte, __pte, __npte) != __pte) |
166 | free_pgtable_page(pt: page); |
167 | else if (IOMMU_PTE_PRESENT(__pte)) |
168 | *updated = true; |
169 | |
170 | continue; |
171 | } |
172 | |
173 | level -= 1; |
174 | pte = get_pgtable_pte(pte: __pte); |
175 | pte = &pte[PM_LEVEL_INDEX(level, iova)]; |
176 | } |
177 | |
178 | /* Tear down existing pte entries */ |
179 | if (IOMMU_PTE_PRESENT(*pte)) { |
180 | u64 *__pte; |
181 | |
182 | *updated = true; |
183 | __pte = get_pgtable_pte(pte: *pte); |
184 | cmpxchg64(pte, *pte, 0ULL); |
185 | if (pg_size == IOMMU_PAGE_SIZE_1G) |
186 | free_pgtable(pt: __pte, level: end_level - 1); |
187 | else if (pg_size == IOMMU_PAGE_SIZE_2M) |
188 | free_pgtable_page(pt: __pte); |
189 | } |
190 | |
191 | return pte; |
192 | } |
193 | |
194 | /* |
195 | * This function checks if there is a PTE for a given dma address. |
196 | * If there is one, it returns the pointer to it. |
197 | */ |
198 | static u64 *fetch_pte(struct amd_io_pgtable *pgtable, |
199 | unsigned long iova, unsigned long *page_size) |
200 | { |
201 | u64 *pte; |
202 | int level; |
203 | |
204 | level = get_pgtable_level() - 1; |
205 | pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)]; |
206 | /* Default page size is 4K */ |
207 | *page_size = PAGE_SIZE; |
208 | |
209 | while (level) { |
210 | /* Not present */ |
211 | if (!IOMMU_PTE_PRESENT(*pte)) |
212 | return NULL; |
213 | |
214 | /* Walk to the next level */ |
215 | pte = get_pgtable_pte(pte: *pte); |
216 | pte = &pte[PM_LEVEL_INDEX(level - 1, iova)]; |
217 | |
218 | /* Large page */ |
219 | if (is_large_pte(pte: *pte)) { |
220 | if (level == PAGE_MODE_3_LEVEL) |
221 | *page_size = IOMMU_PAGE_SIZE_1G; |
222 | else if (level == PAGE_MODE_2_LEVEL) |
223 | *page_size = IOMMU_PAGE_SIZE_2M; |
224 | else |
225 | return NULL; /* Wrongly set PSE bit in PTE */ |
226 | |
227 | break; |
228 | } |
229 | |
230 | level -= 1; |
231 | } |
232 | |
233 | return pte; |
234 | } |
235 | |
236 | static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova, |
237 | phys_addr_t paddr, size_t pgsize, size_t pgcount, |
238 | int prot, gfp_t gfp, size_t *mapped) |
239 | { |
240 | struct protection_domain *pdom = io_pgtable_ops_to_domain(ops); |
241 | struct io_pgtable_cfg *cfg = &pdom->iop.iop.cfg; |
242 | u64 *pte; |
243 | unsigned long map_size; |
244 | unsigned long mapped_size = 0; |
245 | unsigned long o_iova = iova; |
246 | size_t size = pgcount << __ffs(pgsize); |
247 | int ret = 0; |
248 | bool updated = false; |
249 | |
250 | if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize) || !pgcount) |
251 | return -EINVAL; |
252 | |
253 | if (!(prot & IOMMU_PROT_MASK)) |
254 | return -EINVAL; |
255 | |
256 | while (mapped_size < size) { |
257 | map_size = get_alloc_page_size(size: pgsize); |
258 | pte = v2_alloc_pte(nid: pdom->nid, pgd: pdom->iop.pgd, |
259 | iova, pg_size: map_size, gfp, updated: &updated); |
260 | if (!pte) { |
261 | ret = -EINVAL; |
262 | goto out; |
263 | } |
264 | |
265 | *pte = set_pte_attr(paddr, pg_size: map_size, prot); |
266 | |
267 | iova += map_size; |
268 | paddr += map_size; |
269 | mapped_size += map_size; |
270 | } |
271 | |
272 | out: |
273 | if (updated) |
274 | amd_iommu_domain_flush_pages(domain: pdom, address: o_iova, size); |
275 | |
276 | if (mapped) |
277 | *mapped += mapped_size; |
278 | |
279 | return ret; |
280 | } |
281 | |
282 | static unsigned long iommu_v2_unmap_pages(struct io_pgtable_ops *ops, |
283 | unsigned long iova, |
284 | size_t pgsize, size_t pgcount, |
285 | struct iommu_iotlb_gather *gather) |
286 | { |
287 | struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); |
288 | struct io_pgtable_cfg *cfg = &pgtable->iop.cfg; |
289 | unsigned long unmap_size; |
290 | unsigned long unmapped = 0; |
291 | size_t size = pgcount << __ffs(pgsize); |
292 | u64 *pte; |
293 | |
294 | if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) |
295 | return 0; |
296 | |
297 | while (unmapped < size) { |
298 | pte = fetch_pte(pgtable, iova, page_size: &unmap_size); |
299 | if (!pte) |
300 | return unmapped; |
301 | |
302 | *pte = 0ULL; |
303 | |
304 | iova = (iova & ~(unmap_size - 1)) + unmap_size; |
305 | unmapped += unmap_size; |
306 | } |
307 | |
308 | return unmapped; |
309 | } |
310 | |
311 | static phys_addr_t iommu_v2_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) |
312 | { |
313 | struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); |
314 | unsigned long offset_mask, pte_pgsize; |
315 | u64 *pte, __pte; |
316 | |
317 | pte = fetch_pte(pgtable, iova, page_size: &pte_pgsize); |
318 | if (!pte || !IOMMU_PTE_PRESENT(*pte)) |
319 | return 0; |
320 | |
321 | offset_mask = pte_pgsize - 1; |
322 | __pte = __sme_clr(*pte & PM_ADDR_MASK); |
323 | |
324 | return (__pte & ~offset_mask) | (iova & offset_mask); |
325 | } |
326 | |
327 | /* |
328 | * ---------------------------------------------------- |
329 | */ |
330 | static void v2_tlb_flush_all(void *cookie) |
331 | { |
332 | } |
333 | |
334 | static void v2_tlb_flush_walk(unsigned long iova, size_t size, |
335 | size_t granule, void *cookie) |
336 | { |
337 | } |
338 | |
339 | static void v2_tlb_add_page(struct iommu_iotlb_gather *gather, |
340 | unsigned long iova, size_t granule, |
341 | void *cookie) |
342 | { |
343 | } |
344 | |
345 | static const struct iommu_flush_ops v2_flush_ops = { |
346 | .tlb_flush_all = v2_tlb_flush_all, |
347 | .tlb_flush_walk = v2_tlb_flush_walk, |
348 | .tlb_add_page = v2_tlb_add_page, |
349 | }; |
350 | |
351 | static void v2_free_pgtable(struct io_pgtable *iop) |
352 | { |
353 | struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); |
354 | |
355 | if (!pgtable || !pgtable->pgd) |
356 | return; |
357 | |
358 | /* Free page table */ |
359 | free_pgtable(pt: pgtable->pgd, level: get_pgtable_level()); |
360 | pgtable->pgd = NULL; |
361 | } |
362 | |
363 | static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) |
364 | { |
365 | struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); |
366 | struct protection_domain *pdom = (struct protection_domain *)cookie; |
367 | int ias = IOMMU_IN_ADDR_BIT_SIZE; |
368 | |
369 | pgtable->pgd = alloc_pgtable_page(nid: pdom->nid, GFP_ATOMIC); |
370 | if (!pgtable->pgd) |
371 | return NULL; |
372 | |
373 | if (get_pgtable_level() == PAGE_MODE_5_LEVEL) |
374 | ias = 57; |
375 | |
376 | pgtable->iop.ops.map_pages = iommu_v2_map_pages; |
377 | pgtable->iop.ops.unmap_pages = iommu_v2_unmap_pages; |
378 | pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys; |
379 | |
380 | cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2, |
381 | cfg->ias = ias, |
382 | cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, |
383 | cfg->tlb = &v2_flush_ops; |
384 | |
385 | return &pgtable->iop; |
386 | } |
387 | |
388 | struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns = { |
389 | .alloc = v2_alloc_pgtable, |
390 | .free = v2_free_pgtable, |
391 | }; |
392 | |