1 | /* SPDX-License-Identifier: GPL-2.0 */ |
---|---|
2 | #ifndef _ASM_X86_PGTABLE_DEFS_H |
3 | #define _ASM_X86_PGTABLE_DEFS_H |
4 | |
5 | #include <linux/const.h> |
6 | #include <linux/mem_encrypt.h> |
7 | |
8 | #include <asm/page_types.h> |
9 | |
10 | #define _PAGE_BIT_PRESENT 0 /* is present */ |
11 | #define _PAGE_BIT_RW 1 /* writeable */ |
12 | #define _PAGE_BIT_USER 2 /* userspace addressable */ |
13 | #define _PAGE_BIT_PWT 3 /* page write through */ |
14 | #define _PAGE_BIT_PCD 4 /* page cache disabled */ |
15 | #define _PAGE_BIT_ACCESSED 5 /* was accessed (raised by CPU) */ |
16 | #define _PAGE_BIT_DIRTY 6 /* was written to (raised by CPU) */ |
17 | #define _PAGE_BIT_PSE 7 /* 4 MB (or 2MB) page */ |
18 | #define _PAGE_BIT_PAT 7 /* on 4KB pages */ |
19 | #define _PAGE_BIT_GLOBAL 8 /* Global TLB entry PPro+ */ |
20 | #define _PAGE_BIT_SOFTW1 9 /* available for programmer */ |
21 | #define _PAGE_BIT_SOFTW2 10 /* " */ |
22 | #define _PAGE_BIT_SOFTW3 11 /* " */ |
23 | #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */ |
24 | #define _PAGE_BIT_SOFTW4 57 /* available for programmer */ |
25 | #define _PAGE_BIT_SOFTW5 58 /* available for programmer */ |
26 | #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */ |
27 | #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */ |
28 | #define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */ |
29 | #define _PAGE_BIT_PKEY_BIT3 62 /* Protection Keys, bit 4/4 */ |
30 | #define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */ |
31 | |
32 | #define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1 |
33 | #define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1 |
34 | #define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */ |
35 | #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */ |
36 | #define _PAGE_BIT_KERNEL_4K _PAGE_BIT_SOFTW3 /* page must not be converted to large */ |
37 | #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4 |
38 | |
39 | #ifdef CONFIG_X86_64 |
40 | #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */ |
41 | #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */ |
42 | #else |
43 | /* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */ |
44 | #define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */ |
45 | #define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */ |
46 | #endif |
47 | |
48 | /* If _PAGE_BIT_PRESENT is clear, we use these: */ |
49 | /* - if the user mapped it with PROT_NONE; pte_present gives true */ |
50 | #define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL |
51 | |
52 | #define _PAGE_PRESENT (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT) |
53 | #define _PAGE_RW (_AT(pteval_t, 1) << _PAGE_BIT_RW) |
54 | #define _PAGE_USER (_AT(pteval_t, 1) << _PAGE_BIT_USER) |
55 | #define _PAGE_PWT (_AT(pteval_t, 1) << _PAGE_BIT_PWT) |
56 | #define _PAGE_PCD (_AT(pteval_t, 1) << _PAGE_BIT_PCD) |
57 | #define _PAGE_ACCESSED (_AT(pteval_t, 1) << _PAGE_BIT_ACCESSED) |
58 | #define _PAGE_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY) |
59 | #define _PAGE_PSE (_AT(pteval_t, 1) << _PAGE_BIT_PSE) |
60 | #define _PAGE_GLOBAL (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL) |
61 | #define _PAGE_SOFTW1 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1) |
62 | #define _PAGE_SOFTW2 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW2) |
63 | #define _PAGE_SOFTW3 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW3) |
64 | #define _PAGE_PAT (_AT(pteval_t, 1) << _PAGE_BIT_PAT) |
65 | #define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE) |
66 | #define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL) |
67 | #define _PAGE_CPA_TEST (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST) |
68 | #define _PAGE_KERNEL_4K (_AT(pteval_t, 1) << _PAGE_BIT_KERNEL_4K) |
69 | #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS |
70 | #define _PAGE_PKEY_BIT0 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT0) |
71 | #define _PAGE_PKEY_BIT1 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT1) |
72 | #define _PAGE_PKEY_BIT2 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT2) |
73 | #define _PAGE_PKEY_BIT3 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT3) |
74 | #else |
75 | #define _PAGE_PKEY_BIT0 (_AT(pteval_t, 0)) |
76 | #define _PAGE_PKEY_BIT1 (_AT(pteval_t, 0)) |
77 | #define _PAGE_PKEY_BIT2 (_AT(pteval_t, 0)) |
78 | #define _PAGE_PKEY_BIT3 (_AT(pteval_t, 0)) |
79 | #endif |
80 | |
81 | #define _PAGE_PKEY_MASK (_PAGE_PKEY_BIT0 | \ |
82 | _PAGE_PKEY_BIT1 | \ |
83 | _PAGE_PKEY_BIT2 | \ |
84 | _PAGE_PKEY_BIT3) |
85 | |
86 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
87 | #define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED) |
88 | #else |
89 | #define _PAGE_KNL_ERRATUM_MASK 0 |
90 | #endif |
91 | |
92 | #ifdef CONFIG_MEM_SOFT_DIRTY |
93 | #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SOFT_DIRTY) |
94 | #else |
95 | #define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0)) |
96 | #endif |
97 | |
98 | /* |
99 | * Tracking soft dirty bit when a page goes to a swap is tricky. |
100 | * We need a bit which can be stored in pte _and_ not conflict |
101 | * with swap entry format. On x86 bits 1-4 are *not* involved |
102 | * into swap entry computation, but bit 7 is used for thp migration, |
103 | * so we borrow bit 1 for soft dirty tracking. |
104 | * |
105 | * Please note that this bit must be treated as swap dirty page |
106 | * mark if and only if the PTE/PMD has present bit clear! |
107 | */ |
108 | #ifdef CONFIG_MEM_SOFT_DIRTY |
109 | #define _PAGE_SWP_SOFT_DIRTY _PAGE_RW |
110 | #else |
111 | #define _PAGE_SWP_SOFT_DIRTY (_AT(pteval_t, 0)) |
112 | #endif |
113 | |
114 | #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP |
115 | #define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP) |
116 | #define _PAGE_SWP_UFFD_WP _PAGE_USER |
117 | #else |
118 | #define _PAGE_UFFD_WP (_AT(pteval_t, 0)) |
119 | #define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0)) |
120 | #endif |
121 | |
122 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
123 | #define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX) |
124 | #define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP) |
125 | #define _PAGE_SOFTW4 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW4) |
126 | #else |
127 | #define _PAGE_NX (_AT(pteval_t, 0)) |
128 | #define _PAGE_DEVMAP (_AT(pteval_t, 0)) |
129 | #define _PAGE_SOFTW4 (_AT(pteval_t, 0)) |
130 | #endif |
131 | |
132 | /* |
133 | * The hardware requires shadow stack to be Write=0,Dirty=1. However, |
134 | * there are valid cases where the kernel might create read-only PTEs that |
135 | * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In |
136 | * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit, |
137 | * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have |
138 | * (Write=0,SavedDirty=1,Dirty=0) set. |
139 | */ |
140 | #define _PAGE_SAVED_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SAVED_DIRTY) |
141 | |
142 | #define _PAGE_DIRTY_BITS (_PAGE_DIRTY | _PAGE_SAVED_DIRTY) |
143 | |
144 | #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) |
145 | |
146 | #define _PAGE_NOPTISHADOW (_AT(pteval_t, 1) << _PAGE_BIT_NOPTISHADOW) |
147 | |
148 | /* |
149 | * Set of bits not changed in pte_modify. The pte's |
150 | * protection key is treated like _PAGE_RW, for |
151 | * instance, and is *not* included in this mask since |
152 | * pte_modify() does modify it. |
153 | */ |
154 | #define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \ |
155 | _PAGE_SPECIAL | _PAGE_ACCESSED | \ |
156 | _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \ |
157 | _PAGE_DEVMAP | _PAGE_CC | _PAGE_UFFD_WP) |
158 | #define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT) |
159 | #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE) |
160 | |
161 | /* |
162 | * The cache modes defined here are used to translate between pure SW usage |
163 | * and the HW defined cache mode bits and/or PAT entries. |
164 | * |
165 | * The resulting bits for PWT, PCD and PAT should be chosen in a way |
166 | * to have the WB mode at index 0 (all bits clear). This is the default |
167 | * right now and likely would break too much if changed. |
168 | */ |
169 | #ifndef __ASSEMBLER__ |
170 | enum page_cache_mode { |
171 | _PAGE_CACHE_MODE_WB = 0, |
172 | _PAGE_CACHE_MODE_WC = 1, |
173 | _PAGE_CACHE_MODE_UC_MINUS = 2, |
174 | _PAGE_CACHE_MODE_UC = 3, |
175 | _PAGE_CACHE_MODE_WT = 4, |
176 | _PAGE_CACHE_MODE_WP = 5, |
177 | |
178 | _PAGE_CACHE_MODE_NUM = 8 |
179 | }; |
180 | #endif |
181 | |
182 | #define _PAGE_CC (_AT(pteval_t, cc_get_mask())) |
183 | #define _PAGE_ENC (_AT(pteval_t, sme_me_mask)) |
184 | |
185 | #define _PAGE_CACHE_MASK (_PAGE_PWT | _PAGE_PCD | _PAGE_PAT) |
186 | #define _PAGE_LARGE_CACHE_MASK (_PAGE_PWT | _PAGE_PCD | _PAGE_PAT_LARGE) |
187 | |
188 | #define _PAGE_NOCACHE (cachemode2protval(_PAGE_CACHE_MODE_UC)) |
189 | #define _PAGE_CACHE_WP (cachemode2protval(_PAGE_CACHE_MODE_WP)) |
190 | |
191 | #define __PP _PAGE_PRESENT |
192 | #define __RW _PAGE_RW |
193 | #define _USR _PAGE_USER |
194 | #define ___A _PAGE_ACCESSED |
195 | #define ___D _PAGE_DIRTY |
196 | #define ___G _PAGE_GLOBAL |
197 | #define __NX _PAGE_NX |
198 | |
199 | #define _ENC _PAGE_ENC |
200 | #define __WP _PAGE_CACHE_WP |
201 | #define __NC _PAGE_NOCACHE |
202 | #define _PSE _PAGE_PSE |
203 | |
204 | #define pgprot_val(x) ((x).pgprot) |
205 | #define __pgprot(x) ((pgprot_t) { (x) } ) |
206 | #define __pg(x) __pgprot(x) |
207 | |
208 | #define PAGE_NONE __pg( 0| 0| 0|___A| 0| 0| 0|___G) |
209 | #define PAGE_SHARED __pg(__PP|__RW|_USR|___A|__NX| 0| 0| 0) |
210 | #define PAGE_SHARED_EXEC __pg(__PP|__RW|_USR|___A| 0| 0| 0| 0) |
211 | #define PAGE_COPY_NOEXEC __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0) |
212 | #define PAGE_COPY_EXEC __pg(__PP| 0|_USR|___A| 0| 0| 0| 0) |
213 | #define PAGE_COPY __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0) |
214 | #define PAGE_READONLY __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0) |
215 | #define PAGE_READONLY_EXEC __pg(__PP| 0|_USR|___A| 0| 0| 0| 0) |
216 | |
217 | #define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G) |
218 | #define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G) |
219 | |
220 | /* |
221 | * Page tables needs to have Write=1 in order for any lower PTEs to be |
222 | * writable. This includes shadow stack memory (Write=0, Dirty=1) |
223 | */ |
224 | #define _KERNPG_TABLE_NOENC (__PP|__RW| 0|___A| 0|___D| 0| 0) |
225 | #define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC) |
226 | #define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0) |
227 | #define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC) |
228 | |
229 | #define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX| 0| 0|___G) |
230 | #define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0| 0| 0|___G) |
231 | #define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G) |
232 | #define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G) |
233 | #define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC) |
234 | #define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX| 0| 0|___G) |
235 | #define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G) |
236 | #define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G) |
237 | #define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP) |
238 | |
239 | |
240 | #define __PAGE_KERNEL_IO __PAGE_KERNEL |
241 | #define __PAGE_KERNEL_IO_NOCACHE __PAGE_KERNEL_NOCACHE |
242 | |
243 | |
244 | #ifndef __ASSEMBLER__ |
245 | |
246 | #define __PAGE_KERNEL_ENC (__PAGE_KERNEL | _ENC) |
247 | #define __PAGE_KERNEL_ENC_WP (__PAGE_KERNEL_WP | _ENC) |
248 | #define __PAGE_KERNEL_NOENC (__PAGE_KERNEL | 0) |
249 | #define __PAGE_KERNEL_NOENC_WP (__PAGE_KERNEL_WP | 0) |
250 | |
251 | #define __pgprot_mask(x) __pgprot((x) & __default_kernel_pte_mask) |
252 | |
253 | #define PAGE_KERNEL __pgprot_mask(__PAGE_KERNEL | _ENC) |
254 | #define PAGE_KERNEL_NOENC __pgprot_mask(__PAGE_KERNEL | 0) |
255 | #define PAGE_KERNEL_RO __pgprot_mask(__PAGE_KERNEL_RO | _ENC) |
256 | #define PAGE_KERNEL_EXEC __pgprot_mask(__PAGE_KERNEL_EXEC | _ENC) |
257 | #define PAGE_KERNEL_EXEC_NOENC __pgprot_mask(__PAGE_KERNEL_EXEC | 0) |
258 | #define PAGE_KERNEL_ROX __pgprot_mask(__PAGE_KERNEL_ROX | _ENC) |
259 | #define PAGE_KERNEL_NOCACHE __pgprot_mask(__PAGE_KERNEL_NOCACHE | _ENC) |
260 | #define PAGE_KERNEL_LARGE __pgprot_mask(__PAGE_KERNEL_LARGE | _ENC) |
261 | #define PAGE_KERNEL_LARGE_EXEC __pgprot_mask(__PAGE_KERNEL_LARGE_EXEC | _ENC) |
262 | #define PAGE_KERNEL_VVAR __pgprot_mask(__PAGE_KERNEL_VVAR | _ENC) |
263 | |
264 | #define PAGE_KERNEL_IO __pgprot_mask(__PAGE_KERNEL_IO) |
265 | #define PAGE_KERNEL_IO_NOCACHE __pgprot_mask(__PAGE_KERNEL_IO_NOCACHE) |
266 | |
267 | #endif /* __ASSEMBLER__ */ |
268 | |
269 | /* |
270 | * early identity mapping pte attrib macros. |
271 | */ |
272 | #ifdef CONFIG_X86_64 |
273 | #define __PAGE_KERNEL_IDENT_LARGE_EXEC __PAGE_KERNEL_LARGE_EXEC |
274 | #else |
275 | #define PTE_IDENT_ATTR 0x003 /* PRESENT+RW */ |
276 | #define PDE_IDENT_ATTR 0x063 /* PRESENT+RW+DIRTY+ACCESSED */ |
277 | #define PGD_IDENT_ATTR 0x001 /* PRESENT (no other attributes) */ |
278 | #endif |
279 | |
280 | #ifdef CONFIG_X86_32 |
281 | # include <asm/pgtable_32_types.h> |
282 | #else |
283 | # include <asm/pgtable_64_types.h> |
284 | #endif |
285 | |
286 | #ifndef __ASSEMBLER__ |
287 | |
288 | #include <linux/types.h> |
289 | |
290 | /* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */ |
291 | #define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK) |
292 | |
293 | /* |
294 | * Extracts the flags from a (pte|pmd|pud|pgd)val_t |
295 | * This includes the protection key value. |
296 | */ |
297 | #define PTE_FLAGS_MASK (~PTE_PFN_MASK) |
298 | |
299 | typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; |
300 | |
301 | typedef struct { pgdval_t pgd; } pgd_t; |
302 | |
303 | static inline pgprot_t pgprot_nx(pgprot_t prot) |
304 | { |
305 | return __pgprot(pgprot_val(prot) | _PAGE_NX); |
306 | } |
307 | #define pgprot_nx pgprot_nx |
308 | |
309 | #ifdef CONFIG_X86_PAE |
310 | |
311 | /* |
312 | * PHYSICAL_PAGE_MASK might be non-constant when SME is compiled in, so we can't |
313 | * use it here. |
314 | */ |
315 | |
316 | #define PGD_PAE_PAGE_MASK ((signed long)PAGE_MASK) |
317 | #define PGD_PAE_PHYS_MASK (((1ULL << __PHYSICAL_MASK_SHIFT)-1) & PGD_PAE_PAGE_MASK) |
318 | |
319 | /* |
320 | * PAE allows Base Address, P, PWT, PCD and AVL bits to be set in PGD entries. |
321 | * All other bits are Reserved MBZ |
322 | */ |
323 | #define PGD_ALLOWED_BITS (PGD_PAE_PHYS_MASK | _PAGE_PRESENT | \ |
324 | _PAGE_PWT | _PAGE_PCD | \ |
325 | _PAGE_SOFTW1 | _PAGE_SOFTW2 | _PAGE_SOFTW3) |
326 | |
327 | #else |
328 | /* No need to mask any bits for !PAE */ |
329 | #define PGD_ALLOWED_BITS (~0ULL) |
330 | #endif |
331 | |
332 | static inline pgd_t native_make_pgd(pgdval_t val) |
333 | { |
334 | return (pgd_t) { val & PGD_ALLOWED_BITS }; |
335 | } |
336 | |
337 | static inline pgdval_t native_pgd_val(pgd_t pgd) |
338 | { |
339 | return pgd.pgd & PGD_ALLOWED_BITS; |
340 | } |
341 | |
342 | static inline pgdval_t pgd_flags(pgd_t pgd) |
343 | { |
344 | return native_pgd_val(pgd) & PTE_FLAGS_MASK; |
345 | } |
346 | |
347 | #if CONFIG_PGTABLE_LEVELS > 4 |
348 | typedef struct { p4dval_t p4d; } p4d_t; |
349 | |
350 | static inline p4d_t native_make_p4d(pudval_t val) |
351 | { |
352 | return (p4d_t) { val }; |
353 | } |
354 | |
355 | static inline p4dval_t native_p4d_val(p4d_t p4d) |
356 | { |
357 | return p4d.p4d; |
358 | } |
359 | #else |
360 | #include <asm-generic/pgtable-nop4d.h> |
361 | |
362 | static inline p4d_t native_make_p4d(pudval_t val) |
363 | { |
364 | return (p4d_t) { .pgd = native_make_pgd((pgdval_t)val) }; |
365 | } |
366 | |
367 | static inline p4dval_t native_p4d_val(p4d_t p4d) |
368 | { |
369 | return native_pgd_val(p4d.pgd); |
370 | } |
371 | #endif |
372 | |
373 | #if CONFIG_PGTABLE_LEVELS > 3 |
374 | typedef struct { pudval_t pud; } pud_t; |
375 | |
376 | static inline pud_t native_make_pud(pmdval_t val) |
377 | { |
378 | return (pud_t) { val }; |
379 | } |
380 | |
381 | static inline pudval_t native_pud_val(pud_t pud) |
382 | { |
383 | return pud.pud; |
384 | } |
385 | #else |
386 | #include <asm-generic/pgtable-nopud.h> |
387 | |
388 | static inline pud_t native_make_pud(pudval_t val) |
389 | { |
390 | return (pud_t) { .p4d.pgd = native_make_pgd(val) }; |
391 | } |
392 | |
393 | static inline pudval_t native_pud_val(pud_t pud) |
394 | { |
395 | return native_pgd_val(pud.p4d.pgd); |
396 | } |
397 | #endif |
398 | |
399 | #if CONFIG_PGTABLE_LEVELS > 2 |
400 | static inline pmd_t native_make_pmd(pmdval_t val) |
401 | { |
402 | return (pmd_t) { .pmd = val }; |
403 | } |
404 | |
405 | static inline pmdval_t native_pmd_val(pmd_t pmd) |
406 | { |
407 | return pmd.pmd; |
408 | } |
409 | #else |
410 | #include <asm-generic/pgtable-nopmd.h> |
411 | |
412 | static inline pmd_t native_make_pmd(pmdval_t val) |
413 | { |
414 | return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) }; |
415 | } |
416 | |
417 | static inline pmdval_t native_pmd_val(pmd_t pmd) |
418 | { |
419 | return native_pgd_val(pmd.pud.p4d.pgd); |
420 | } |
421 | #endif |
422 | |
423 | static inline p4dval_t p4d_pfn_mask(p4d_t p4d) |
424 | { |
425 | /* No 512 GiB huge pages yet */ |
426 | return PTE_PFN_MASK; |
427 | } |
428 | |
429 | static inline p4dval_t p4d_flags_mask(p4d_t p4d) |
430 | { |
431 | return ~p4d_pfn_mask(p4d); |
432 | } |
433 | |
434 | static inline p4dval_t p4d_flags(p4d_t p4d) |
435 | { |
436 | return native_p4d_val(p4d) & p4d_flags_mask(p4d); |
437 | } |
438 | |
439 | static inline pudval_t pud_pfn_mask(pud_t pud) |
440 | { |
441 | if (native_pud_val(pud) & _PAGE_PSE) |
442 | return PHYSICAL_PUD_PAGE_MASK; |
443 | else |
444 | return PTE_PFN_MASK; |
445 | } |
446 | |
447 | static inline pudval_t pud_flags_mask(pud_t pud) |
448 | { |
449 | return ~pud_pfn_mask(pud); |
450 | } |
451 | |
452 | static inline pudval_t pud_flags(pud_t pud) |
453 | { |
454 | return native_pud_val(pud) & pud_flags_mask(pud); |
455 | } |
456 | |
457 | static inline pmdval_t pmd_pfn_mask(pmd_t pmd) |
458 | { |
459 | if (native_pmd_val(pmd) & _PAGE_PSE) |
460 | return PHYSICAL_PMD_PAGE_MASK; |
461 | else |
462 | return PTE_PFN_MASK; |
463 | } |
464 | |
465 | static inline pmdval_t pmd_flags_mask(pmd_t pmd) |
466 | { |
467 | return ~pmd_pfn_mask(pmd); |
468 | } |
469 | |
470 | static inline pmdval_t pmd_flags(pmd_t pmd) |
471 | { |
472 | return native_pmd_val(pmd) & pmd_flags_mask(pmd); |
473 | } |
474 | |
475 | static inline pte_t native_make_pte(pteval_t val) |
476 | { |
477 | return (pte_t) { .pte = val }; |
478 | } |
479 | |
480 | static inline pteval_t native_pte_val(pte_t pte) |
481 | { |
482 | return pte.pte; |
483 | } |
484 | |
485 | static inline pteval_t pte_flags(pte_t pte) |
486 | { |
487 | return native_pte_val(pte) & PTE_FLAGS_MASK; |
488 | } |
489 | |
490 | #define __pte2cm_idx(cb) \ |
491 | ((((cb) >> (_PAGE_BIT_PAT - 2)) & 4) | \ |
492 | (((cb) >> (_PAGE_BIT_PCD - 1)) & 2) | \ |
493 | (((cb) >> _PAGE_BIT_PWT) & 1)) |
494 | #define __cm_idx2pte(i) \ |
495 | ((((i) & 4) << (_PAGE_BIT_PAT - 2)) | \ |
496 | (((i) & 2) << (_PAGE_BIT_PCD - 1)) | \ |
497 | (((i) & 1) << _PAGE_BIT_PWT)) |
498 | |
499 | unsigned long cachemode2protval(enum page_cache_mode pcm); |
500 | |
501 | static inline pgprotval_t protval_4k_2_large(pgprotval_t val) |
502 | { |
503 | return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | |
504 | ((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); |
505 | } |
506 | static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot) |
507 | { |
508 | return __pgprot(protval_4k_2_large(pgprot_val(pgprot))); |
509 | } |
510 | static inline pgprotval_t protval_large_2_4k(pgprotval_t val) |
511 | { |
512 | return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | |
513 | ((val & _PAGE_PAT_LARGE) >> |
514 | (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT)); |
515 | } |
516 | static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot) |
517 | { |
518 | return __pgprot(protval_large_2_4k(pgprot_val(pgprot))); |
519 | } |
520 | |
521 | |
522 | typedef struct page *pgtable_t; |
523 | |
524 | extern pteval_t __supported_pte_mask; |
525 | extern pteval_t __default_kernel_pte_mask; |
526 | |
527 | #define pgprot_writecombine pgprot_writecombine |
528 | extern pgprot_t pgprot_writecombine(pgprot_t prot); |
529 | |
530 | #define pgprot_writethrough pgprot_writethrough |
531 | extern pgprot_t pgprot_writethrough(pgprot_t prot); |
532 | |
533 | /* Indicate that x86 has its own track and untrack pfn vma functions */ |
534 | #define __HAVE_PFNMAP_TRACKING |
535 | |
536 | #define __HAVE_PHYS_MEM_ACCESS_PROT |
537 | struct file; |
538 | pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
539 | unsigned long size, pgprot_t vma_prot); |
540 | |
541 | /* Install a pte for a particular vaddr in kernel space. */ |
542 | void set_pte_vaddr(unsigned long vaddr, pte_t pte); |
543 | |
544 | #ifdef CONFIG_X86_32 |
545 | extern void native_pagetable_init(void); |
546 | #else |
547 | #define native_pagetable_init paging_init |
548 | #endif |
549 | |
550 | enum pg_level { |
551 | PG_LEVEL_NONE, |
552 | PG_LEVEL_4K, |
553 | PG_LEVEL_2M, |
554 | PG_LEVEL_1G, |
555 | PG_LEVEL_512G, |
556 | PG_LEVEL_256T, |
557 | PG_LEVEL_NUM |
558 | }; |
559 | |
560 | #ifdef CONFIG_PROC_FS |
561 | extern void update_page_count(int level, unsigned long pages); |
562 | #else |
563 | static inline void update_page_count(int level, unsigned long pages) { } |
564 | #endif |
565 | |
566 | /* |
567 | * Helper function that returns the kernel pagetable entry controlling |
568 | * the virtual address 'address'. NULL means no pagetable entry present. |
569 | * NOTE: the return type is pte_t but if the pmd is PSE then we return it |
570 | * as a pte too. |
571 | */ |
572 | extern pte_t *lookup_address(unsigned long address, unsigned int *level); |
573 | extern pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address, |
574 | unsigned int *level); |
575 | pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address, |
576 | unsigned int *level, bool *nx, bool *rw); |
577 | extern pmd_t *lookup_pmd_address(unsigned long address); |
578 | extern phys_addr_t slow_virt_to_phys(void *__address); |
579 | extern int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, |
580 | unsigned long address, |
581 | unsigned numpages, |
582 | unsigned long page_flags); |
583 | extern int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address, |
584 | unsigned long numpages); |
585 | #endif /* !__ASSEMBLER__ */ |
586 | |
587 | #endif /* _ASM_X86_PGTABLE_DEFS_H */ |
588 |
Definitions
- page_cache_mode
- pgprot
- pgprot_nx
- native_make_pgd
- native_pgd_val
- pgd_flags
- native_make_p4d
- native_p4d_val
- native_make_pud
- native_pud_val
- native_make_pmd
- native_pmd_val
- p4d_pfn_mask
- p4d_flags_mask
- p4d_flags
- pud_pfn_mask
- pud_flags_mask
- pud_flags
- pmd_pfn_mask
- pmd_flags_mask
- pmd_flags
- native_make_pte
- native_pte_val
- pte_flags
- protval_4k_2_large
- pgprot_4k_2_large
- protval_large_2_4k
- pgprot_large_2_4k
Improve your Profiling and Debugging skills
Find out more