1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_PGTABLE_DEFS_H
3#define _ASM_X86_PGTABLE_DEFS_H
4
5#include <linux/const.h>
6#include <linux/mem_encrypt.h>
7
8#include <asm/page_types.h>
9
10#define _PAGE_BIT_PRESENT 0 /* is present */
11#define _PAGE_BIT_RW 1 /* writeable */
12#define _PAGE_BIT_USER 2 /* userspace addressable */
13#define _PAGE_BIT_PWT 3 /* page write through */
14#define _PAGE_BIT_PCD 4 /* page cache disabled */
15#define _PAGE_BIT_ACCESSED 5 /* was accessed (raised by CPU) */
16#define _PAGE_BIT_DIRTY 6 /* was written to (raised by CPU) */
17#define _PAGE_BIT_PSE 7 /* 4 MB (or 2MB) page */
18#define _PAGE_BIT_PAT 7 /* on 4KB pages */
19#define _PAGE_BIT_GLOBAL 8 /* Global TLB entry PPro+ */
20#define _PAGE_BIT_SOFTW1 9 /* available for programmer */
21#define _PAGE_BIT_SOFTW2 10 /* " */
22#define _PAGE_BIT_SOFTW3 11 /* " */
23#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */
24#define _PAGE_BIT_SOFTW4 57 /* available for programmer */
25#define _PAGE_BIT_SOFTW5 58 /* available for programmer */
26#define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */
27#define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */
28#define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */
29#define _PAGE_BIT_PKEY_BIT3 62 /* Protection Keys, bit 4/4 */
30#define _PAGE_BIT_NX 63 /* No execute: only valid after cpuid check */
31
32#define _PAGE_BIT_SPECIAL _PAGE_BIT_SOFTW1
33#define _PAGE_BIT_CPA_TEST _PAGE_BIT_SOFTW1
34#define _PAGE_BIT_UFFD_WP _PAGE_BIT_SOFTW2 /* userfaultfd wrprotected */
35#define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */
36#define _PAGE_BIT_KERNEL_4K _PAGE_BIT_SOFTW3 /* page must not be converted to large */
37#define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4
38
39#ifdef CONFIG_X86_64
40#define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit (leaf) */
41#define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW5 /* No PTI shadow (root PGD) */
42#else
43/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */
44#define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit (leaf) */
45#define _PAGE_BIT_NOPTISHADOW _PAGE_BIT_SOFTW2 /* No PTI shadow (root PGD) */
46#endif
47
48/* If _PAGE_BIT_PRESENT is clear, we use these: */
49/* - if the user mapped it with PROT_NONE; pte_present gives true */
50#define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL
51
52#define _PAGE_PRESENT (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT)
53#define _PAGE_RW (_AT(pteval_t, 1) << _PAGE_BIT_RW)
54#define _PAGE_USER (_AT(pteval_t, 1) << _PAGE_BIT_USER)
55#define _PAGE_PWT (_AT(pteval_t, 1) << _PAGE_BIT_PWT)
56#define _PAGE_PCD (_AT(pteval_t, 1) << _PAGE_BIT_PCD)
57#define _PAGE_ACCESSED (_AT(pteval_t, 1) << _PAGE_BIT_ACCESSED)
58#define _PAGE_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY)
59#define _PAGE_PSE (_AT(pteval_t, 1) << _PAGE_BIT_PSE)
60#define _PAGE_GLOBAL (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL)
61#define _PAGE_SOFTW1 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1)
62#define _PAGE_SOFTW2 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW2)
63#define _PAGE_SOFTW3 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW3)
64#define _PAGE_PAT (_AT(pteval_t, 1) << _PAGE_BIT_PAT)
65#define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE)
66#define _PAGE_SPECIAL (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL)
67#define _PAGE_CPA_TEST (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST)
68#define _PAGE_KERNEL_4K (_AT(pteval_t, 1) << _PAGE_BIT_KERNEL_4K)
69#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
70#define _PAGE_PKEY_BIT0 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT0)
71#define _PAGE_PKEY_BIT1 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT1)
72#define _PAGE_PKEY_BIT2 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT2)
73#define _PAGE_PKEY_BIT3 (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT3)
74#else
75#define _PAGE_PKEY_BIT0 (_AT(pteval_t, 0))
76#define _PAGE_PKEY_BIT1 (_AT(pteval_t, 0))
77#define _PAGE_PKEY_BIT2 (_AT(pteval_t, 0))
78#define _PAGE_PKEY_BIT3 (_AT(pteval_t, 0))
79#endif
80
81#define _PAGE_PKEY_MASK (_PAGE_PKEY_BIT0 | \
82 _PAGE_PKEY_BIT1 | \
83 _PAGE_PKEY_BIT2 | \
84 _PAGE_PKEY_BIT3)
85
86#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
87#define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED)
88#else
89#define _PAGE_KNL_ERRATUM_MASK 0
90#endif
91
92#ifdef CONFIG_MEM_SOFT_DIRTY
93#define _PAGE_SOFT_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SOFT_DIRTY)
94#else
95#define _PAGE_SOFT_DIRTY (_AT(pteval_t, 0))
96#endif
97
98/*
99 * Tracking soft dirty bit when a page goes to a swap is tricky.
100 * We need a bit which can be stored in pte _and_ not conflict
101 * with swap entry format. On x86 bits 1-4 are *not* involved
102 * into swap entry computation, but bit 7 is used for thp migration,
103 * so we borrow bit 1 for soft dirty tracking.
104 *
105 * Please note that this bit must be treated as swap dirty page
106 * mark if and only if the PTE/PMD has present bit clear!
107 */
108#ifdef CONFIG_MEM_SOFT_DIRTY
109#define _PAGE_SWP_SOFT_DIRTY _PAGE_RW
110#else
111#define _PAGE_SWP_SOFT_DIRTY (_AT(pteval_t, 0))
112#endif
113
114#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
115#define _PAGE_UFFD_WP (_AT(pteval_t, 1) << _PAGE_BIT_UFFD_WP)
116#define _PAGE_SWP_UFFD_WP _PAGE_USER
117#else
118#define _PAGE_UFFD_WP (_AT(pteval_t, 0))
119#define _PAGE_SWP_UFFD_WP (_AT(pteval_t, 0))
120#endif
121
122#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
123#define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX)
124#define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP)
125#define _PAGE_SOFTW4 (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW4)
126#else
127#define _PAGE_NX (_AT(pteval_t, 0))
128#define _PAGE_DEVMAP (_AT(pteval_t, 0))
129#define _PAGE_SOFTW4 (_AT(pteval_t, 0))
130#endif
131
132/*
133 * The hardware requires shadow stack to be Write=0,Dirty=1. However,
134 * there are valid cases where the kernel might create read-only PTEs that
135 * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In
136 * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit,
137 * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have
138 * (Write=0,SavedDirty=1,Dirty=0) set.
139 */
140#define _PAGE_SAVED_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SAVED_DIRTY)
141
142#define _PAGE_DIRTY_BITS (_PAGE_DIRTY | _PAGE_SAVED_DIRTY)
143
144#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
145
146#define _PAGE_NOPTISHADOW (_AT(pteval_t, 1) << _PAGE_BIT_NOPTISHADOW)
147
148/*
149 * Set of bits not changed in pte_modify. The pte's
150 * protection key is treated like _PAGE_RW, for
151 * instance, and is *not* included in this mask since
152 * pte_modify() does modify it.
153 */
154#define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \
155 _PAGE_SPECIAL | _PAGE_ACCESSED | \
156 _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \
157 _PAGE_DEVMAP | _PAGE_CC | _PAGE_UFFD_WP)
158#define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT)
159#define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE)
160
161/*
162 * The cache modes defined here are used to translate between pure SW usage
163 * and the HW defined cache mode bits and/or PAT entries.
164 *
165 * The resulting bits for PWT, PCD and PAT should be chosen in a way
166 * to have the WB mode at index 0 (all bits clear). This is the default
167 * right now and likely would break too much if changed.
168 */
169#ifndef __ASSEMBLER__
170enum page_cache_mode {
171 _PAGE_CACHE_MODE_WB = 0,
172 _PAGE_CACHE_MODE_WC = 1,
173 _PAGE_CACHE_MODE_UC_MINUS = 2,
174 _PAGE_CACHE_MODE_UC = 3,
175 _PAGE_CACHE_MODE_WT = 4,
176 _PAGE_CACHE_MODE_WP = 5,
177
178 _PAGE_CACHE_MODE_NUM = 8
179};
180#endif
181
182#define _PAGE_CC (_AT(pteval_t, cc_get_mask()))
183#define _PAGE_ENC (_AT(pteval_t, sme_me_mask))
184
185#define _PAGE_CACHE_MASK (_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)
186#define _PAGE_LARGE_CACHE_MASK (_PAGE_PWT | _PAGE_PCD | _PAGE_PAT_LARGE)
187
188#define _PAGE_NOCACHE (cachemode2protval(_PAGE_CACHE_MODE_UC))
189#define _PAGE_CACHE_WP (cachemode2protval(_PAGE_CACHE_MODE_WP))
190
191#define __PP _PAGE_PRESENT
192#define __RW _PAGE_RW
193#define _USR _PAGE_USER
194#define ___A _PAGE_ACCESSED
195#define ___D _PAGE_DIRTY
196#define ___G _PAGE_GLOBAL
197#define __NX _PAGE_NX
198
199#define _ENC _PAGE_ENC
200#define __WP _PAGE_CACHE_WP
201#define __NC _PAGE_NOCACHE
202#define _PSE _PAGE_PSE
203
204#define pgprot_val(x) ((x).pgprot)
205#define __pgprot(x) ((pgprot_t) { (x) } )
206#define __pg(x) __pgprot(x)
207
208#define PAGE_NONE __pg( 0| 0| 0|___A| 0| 0| 0|___G)
209#define PAGE_SHARED __pg(__PP|__RW|_USR|___A|__NX| 0| 0| 0)
210#define PAGE_SHARED_EXEC __pg(__PP|__RW|_USR|___A| 0| 0| 0| 0)
211#define PAGE_COPY_NOEXEC __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0)
212#define PAGE_COPY_EXEC __pg(__PP| 0|_USR|___A| 0| 0| 0| 0)
213#define PAGE_COPY __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0)
214#define PAGE_READONLY __pg(__PP| 0|_USR|___A|__NX| 0| 0| 0)
215#define PAGE_READONLY_EXEC __pg(__PP| 0|_USR|___A| 0| 0| 0| 0)
216
217#define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)
218#define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)
219
220/*
221 * Page tables needs to have Write=1 in order for any lower PTEs to be
222 * writable. This includes shadow stack memory (Write=0, Dirty=1)
223 */
224#define _KERNPG_TABLE_NOENC (__PP|__RW| 0|___A| 0|___D| 0| 0)
225#define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC)
226#define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0)
227#define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC)
228
229#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX| 0| 0|___G)
230#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0| 0| 0|___G)
231#define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)
232#define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)
233#define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC)
234#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX| 0| 0|___G)
235#define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G)
236#define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G)
237#define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP)
238
239
240#define __PAGE_KERNEL_IO __PAGE_KERNEL
241#define __PAGE_KERNEL_IO_NOCACHE __PAGE_KERNEL_NOCACHE
242
243
244#ifndef __ASSEMBLER__
245
246#define __PAGE_KERNEL_ENC (__PAGE_KERNEL | _ENC)
247#define __PAGE_KERNEL_ENC_WP (__PAGE_KERNEL_WP | _ENC)
248#define __PAGE_KERNEL_NOENC (__PAGE_KERNEL | 0)
249#define __PAGE_KERNEL_NOENC_WP (__PAGE_KERNEL_WP | 0)
250
251#define __pgprot_mask(x) __pgprot((x) & __default_kernel_pte_mask)
252
253#define PAGE_KERNEL __pgprot_mask(__PAGE_KERNEL | _ENC)
254#define PAGE_KERNEL_NOENC __pgprot_mask(__PAGE_KERNEL | 0)
255#define PAGE_KERNEL_RO __pgprot_mask(__PAGE_KERNEL_RO | _ENC)
256#define PAGE_KERNEL_EXEC __pgprot_mask(__PAGE_KERNEL_EXEC | _ENC)
257#define PAGE_KERNEL_EXEC_NOENC __pgprot_mask(__PAGE_KERNEL_EXEC | 0)
258#define PAGE_KERNEL_ROX __pgprot_mask(__PAGE_KERNEL_ROX | _ENC)
259#define PAGE_KERNEL_NOCACHE __pgprot_mask(__PAGE_KERNEL_NOCACHE | _ENC)
260#define PAGE_KERNEL_LARGE __pgprot_mask(__PAGE_KERNEL_LARGE | _ENC)
261#define PAGE_KERNEL_LARGE_EXEC __pgprot_mask(__PAGE_KERNEL_LARGE_EXEC | _ENC)
262#define PAGE_KERNEL_VVAR __pgprot_mask(__PAGE_KERNEL_VVAR | _ENC)
263
264#define PAGE_KERNEL_IO __pgprot_mask(__PAGE_KERNEL_IO)
265#define PAGE_KERNEL_IO_NOCACHE __pgprot_mask(__PAGE_KERNEL_IO_NOCACHE)
266
267#endif /* __ASSEMBLER__ */
268
269/*
270 * early identity mapping pte attrib macros.
271 */
272#ifdef CONFIG_X86_64
273#define __PAGE_KERNEL_IDENT_LARGE_EXEC __PAGE_KERNEL_LARGE_EXEC
274#else
275#define PTE_IDENT_ATTR 0x003 /* PRESENT+RW */
276#define PDE_IDENT_ATTR 0x063 /* PRESENT+RW+DIRTY+ACCESSED */
277#define PGD_IDENT_ATTR 0x001 /* PRESENT (no other attributes) */
278#endif
279
280#ifdef CONFIG_X86_32
281# include <asm/pgtable_32_types.h>
282#else
283# include <asm/pgtable_64_types.h>
284#endif
285
286#ifndef __ASSEMBLER__
287
288#include <linux/types.h>
289
290/* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */
291#define PTE_PFN_MASK ((pteval_t)PHYSICAL_PAGE_MASK)
292
293/*
294 * Extracts the flags from a (pte|pmd|pud|pgd)val_t
295 * This includes the protection key value.
296 */
297#define PTE_FLAGS_MASK (~PTE_PFN_MASK)
298
299typedef struct pgprot { pgprotval_t pgprot; } pgprot_t;
300
301typedef struct { pgdval_t pgd; } pgd_t;
302
303static inline pgprot_t pgprot_nx(pgprot_t prot)
304{
305 return __pgprot(pgprot_val(prot) | _PAGE_NX);
306}
307#define pgprot_nx pgprot_nx
308
309#ifdef CONFIG_X86_PAE
310
311/*
312 * PHYSICAL_PAGE_MASK might be non-constant when SME is compiled in, so we can't
313 * use it here.
314 */
315
316#define PGD_PAE_PAGE_MASK ((signed long)PAGE_MASK)
317#define PGD_PAE_PHYS_MASK (((1ULL << __PHYSICAL_MASK_SHIFT)-1) & PGD_PAE_PAGE_MASK)
318
319/*
320 * PAE allows Base Address, P, PWT, PCD and AVL bits to be set in PGD entries.
321 * All other bits are Reserved MBZ
322 */
323#define PGD_ALLOWED_BITS (PGD_PAE_PHYS_MASK | _PAGE_PRESENT | \
324 _PAGE_PWT | _PAGE_PCD | \
325 _PAGE_SOFTW1 | _PAGE_SOFTW2 | _PAGE_SOFTW3)
326
327#else
328/* No need to mask any bits for !PAE */
329#define PGD_ALLOWED_BITS (~0ULL)
330#endif
331
332static inline pgd_t native_make_pgd(pgdval_t val)
333{
334 return (pgd_t) { val & PGD_ALLOWED_BITS };
335}
336
337static inline pgdval_t native_pgd_val(pgd_t pgd)
338{
339 return pgd.pgd & PGD_ALLOWED_BITS;
340}
341
342static inline pgdval_t pgd_flags(pgd_t pgd)
343{
344 return native_pgd_val(pgd) & PTE_FLAGS_MASK;
345}
346
347#if CONFIG_PGTABLE_LEVELS > 4
348typedef struct { p4dval_t p4d; } p4d_t;
349
350static inline p4d_t native_make_p4d(pudval_t val)
351{
352 return (p4d_t) { val };
353}
354
355static inline p4dval_t native_p4d_val(p4d_t p4d)
356{
357 return p4d.p4d;
358}
359#else
360#include <asm-generic/pgtable-nop4d.h>
361
362static inline p4d_t native_make_p4d(pudval_t val)
363{
364 return (p4d_t) { .pgd = native_make_pgd((pgdval_t)val) };
365}
366
367static inline p4dval_t native_p4d_val(p4d_t p4d)
368{
369 return native_pgd_val(p4d.pgd);
370}
371#endif
372
373#if CONFIG_PGTABLE_LEVELS > 3
374typedef struct { pudval_t pud; } pud_t;
375
376static inline pud_t native_make_pud(pmdval_t val)
377{
378 return (pud_t) { val };
379}
380
381static inline pudval_t native_pud_val(pud_t pud)
382{
383 return pud.pud;
384}
385#else
386#include <asm-generic/pgtable-nopud.h>
387
388static inline pud_t native_make_pud(pudval_t val)
389{
390 return (pud_t) { .p4d.pgd = native_make_pgd(val) };
391}
392
393static inline pudval_t native_pud_val(pud_t pud)
394{
395 return native_pgd_val(pud.p4d.pgd);
396}
397#endif
398
399#if CONFIG_PGTABLE_LEVELS > 2
400static inline pmd_t native_make_pmd(pmdval_t val)
401{
402 return (pmd_t) { .pmd = val };
403}
404
405static inline pmdval_t native_pmd_val(pmd_t pmd)
406{
407 return pmd.pmd;
408}
409#else
410#include <asm-generic/pgtable-nopmd.h>
411
412static inline pmd_t native_make_pmd(pmdval_t val)
413{
414 return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) };
415}
416
417static inline pmdval_t native_pmd_val(pmd_t pmd)
418{
419 return native_pgd_val(pmd.pud.p4d.pgd);
420}
421#endif
422
423static inline p4dval_t p4d_pfn_mask(p4d_t p4d)
424{
425 /* No 512 GiB huge pages yet */
426 return PTE_PFN_MASK;
427}
428
429static inline p4dval_t p4d_flags_mask(p4d_t p4d)
430{
431 return ~p4d_pfn_mask(p4d);
432}
433
434static inline p4dval_t p4d_flags(p4d_t p4d)
435{
436 return native_p4d_val(p4d) & p4d_flags_mask(p4d);
437}
438
439static inline pudval_t pud_pfn_mask(pud_t pud)
440{
441 if (native_pud_val(pud) & _PAGE_PSE)
442 return PHYSICAL_PUD_PAGE_MASK;
443 else
444 return PTE_PFN_MASK;
445}
446
447static inline pudval_t pud_flags_mask(pud_t pud)
448{
449 return ~pud_pfn_mask(pud);
450}
451
452static inline pudval_t pud_flags(pud_t pud)
453{
454 return native_pud_val(pud) & pud_flags_mask(pud);
455}
456
457static inline pmdval_t pmd_pfn_mask(pmd_t pmd)
458{
459 if (native_pmd_val(pmd) & _PAGE_PSE)
460 return PHYSICAL_PMD_PAGE_MASK;
461 else
462 return PTE_PFN_MASK;
463}
464
465static inline pmdval_t pmd_flags_mask(pmd_t pmd)
466{
467 return ~pmd_pfn_mask(pmd);
468}
469
470static inline pmdval_t pmd_flags(pmd_t pmd)
471{
472 return native_pmd_val(pmd) & pmd_flags_mask(pmd);
473}
474
475static inline pte_t native_make_pte(pteval_t val)
476{
477 return (pte_t) { .pte = val };
478}
479
480static inline pteval_t native_pte_val(pte_t pte)
481{
482 return pte.pte;
483}
484
485static inline pteval_t pte_flags(pte_t pte)
486{
487 return native_pte_val(pte) & PTE_FLAGS_MASK;
488}
489
490#define __pte2cm_idx(cb) \
491 ((((cb) >> (_PAGE_BIT_PAT - 2)) & 4) | \
492 (((cb) >> (_PAGE_BIT_PCD - 1)) & 2) | \
493 (((cb) >> _PAGE_BIT_PWT) & 1))
494#define __cm_idx2pte(i) \
495 ((((i) & 4) << (_PAGE_BIT_PAT - 2)) | \
496 (((i) & 2) << (_PAGE_BIT_PCD - 1)) | \
497 (((i) & 1) << _PAGE_BIT_PWT))
498
499unsigned long cachemode2protval(enum page_cache_mode pcm);
500
501static inline pgprotval_t protval_4k_2_large(pgprotval_t val)
502{
503 return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
504 ((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
505}
506static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot)
507{
508 return __pgprot(protval_4k_2_large(pgprot_val(pgprot)));
509}
510static inline pgprotval_t protval_large_2_4k(pgprotval_t val)
511{
512 return (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
513 ((val & _PAGE_PAT_LARGE) >>
514 (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
515}
516static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot)
517{
518 return __pgprot(protval_large_2_4k(pgprot_val(pgprot)));
519}
520
521
522typedef struct page *pgtable_t;
523
524extern pteval_t __supported_pte_mask;
525extern pteval_t __default_kernel_pte_mask;
526
527#define pgprot_writecombine pgprot_writecombine
528extern pgprot_t pgprot_writecombine(pgprot_t prot);
529
530#define pgprot_writethrough pgprot_writethrough
531extern pgprot_t pgprot_writethrough(pgprot_t prot);
532
533/* Indicate that x86 has its own track and untrack pfn vma functions */
534#define __HAVE_PFNMAP_TRACKING
535
536#define __HAVE_PHYS_MEM_ACCESS_PROT
537struct file;
538pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
539 unsigned long size, pgprot_t vma_prot);
540
541/* Install a pte for a particular vaddr in kernel space. */
542void set_pte_vaddr(unsigned long vaddr, pte_t pte);
543
544#ifdef CONFIG_X86_32
545extern void native_pagetable_init(void);
546#else
547#define native_pagetable_init paging_init
548#endif
549
550enum pg_level {
551 PG_LEVEL_NONE,
552 PG_LEVEL_4K,
553 PG_LEVEL_2M,
554 PG_LEVEL_1G,
555 PG_LEVEL_512G,
556 PG_LEVEL_256T,
557 PG_LEVEL_NUM
558};
559
560#ifdef CONFIG_PROC_FS
561extern void update_page_count(int level, unsigned long pages);
562#else
563static inline void update_page_count(int level, unsigned long pages) { }
564#endif
565
566/*
567 * Helper function that returns the kernel pagetable entry controlling
568 * the virtual address 'address'. NULL means no pagetable entry present.
569 * NOTE: the return type is pte_t but if the pmd is PSE then we return it
570 * as a pte too.
571 */
572extern pte_t *lookup_address(unsigned long address, unsigned int *level);
573extern pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
574 unsigned int *level);
575pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
576 unsigned int *level, bool *nx, bool *rw);
577extern pmd_t *lookup_pmd_address(unsigned long address);
578extern phys_addr_t slow_virt_to_phys(void *__address);
579extern int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn,
580 unsigned long address,
581 unsigned numpages,
582 unsigned long page_flags);
583extern int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
584 unsigned long numpages);
585#endif /* !__ASSEMBLER__ */
586
587#endif /* _ASM_X86_PGTABLE_DEFS_H */
588

Provided by KDAB

Privacy Policy
Improve your Profiling and Debugging skills
Find out more

source code of linux/arch/x86/include/asm/pgtable_types.h